The start of release changes, the changes that turn JDK N
+into JDK (N+1), are primarily small updates to various files
+along with new files to store symbol information to allow
+javac --release N ... to run on JDK (N+1).
+
The updates include changes to files holding meta-data about the
+release, files under the src directory for API and tooling
+updates, and incidental updates under the test
+directory.
+
Details and file updates
+
As a matter of policy, there are a number of semantically distinct
+concepts which get incremented separately at the start of a new
+release:
+
+
Feature value of Runtime.version()
+
Highest source version modeled by
+javax.lang.model.SourceVersion
+
Highest class file format major version recognized by the
+platform
+
Highest
+-source/-target/--release
+argument recognized by javac and related tools
+
+
The expected file updates are listed below. Additional files may need
+to be updated for a particular release.
+
Meta-data files
+
+
jcheck/conf: update meta-data used by
+jcheck and the Skara tooling
+
make/conf/version-numbers.conf: update to meta-data
+used in the build
+
+
src files
+
+
src/hotspot/share/classfile/classFileParser.cpp: add a
+#define for the new version
+
src/java.base/share/classes/java/lang/classfile/ClassFile.java:
+add a constant for the new class file format version
+
src/java.base/share/classes/java/lang/reflect/ClassFileFormatVersion.java:
+add an enum constant for the new class file format
+version
+
src/java.compiler/share/classes/javax/lang/model/SourceVersion.java:
+add an enum constant for the new source version
+
src/java.compiler/share/classes/javax/lang/model/util/*
+visitors: Update @SupportedSourceVersion annotations to
+latest value. Note this update is done in lieu of introducing another
+set of visitors for each Java SE release.
+
src/jdk.compiler/share/classes/com/sun/tools/javac/code/Source.java:
+add an enum constant for the new source version internal to
+javac
+
src/jdk.compiler/share/classes/com/sun/tools/javac/jvm/ClassFile.java:
+add an enum constant for the new class file format version
+internal to javac
+
src/jdk.compiler/share/classes/com/sun/tools/javac/jvm/Target.java:
+add an enum constant for the new target version internal to
+javac
+
src/jdk.compiler/share/classes/com/sun/tools/javac/processing/PrintingProcessor.java
+update printing processor to support the new source version
+
The symbol information for --release is stored as new
+text files in the src/jdk.compiler/share/data/symbols
+directory, one file per module. The README file in that directory
+contains directions on how to create the files.
+
+
test files
+
+
test/langtools/tools/javac/api/TestGetSourceVersions.java:
+add new SourceVersion constant to test matrix.
+
test/langtools/tools/javac/classfiles/ClassVersionChecker.java:
+add new enum constant for the new class file version
+
test/langtools/tools/javac/lib/JavacTestingAbstractProcessor.java
+update annotation processor extended by javac tests to
+cover the new source version
+
test/langtools/tools/javac/preview/classReaderTest/Client.nopreview.out
+and
+test/langtools/tools/javac/preview/classReaderTest/Client.preview.out:
+update expected messages for preview errors and warnings
+
+
+
diff --git a/doc/starting-next-release.md b/doc/starting-next-release.md
new file mode 100644
index 00000000000..10bc364a3e4
--- /dev/null
+++ b/doc/starting-next-release.md
@@ -0,0 +1,68 @@
+% Explanation of start of release changes
+
+## Overview
+
+The start of release changes, the changes that turn JDK _N_ into JDK
+(_N_+1), are primarily small updates to various files along with new files to
+store symbol information to allow `javac --release N ...` to run on
+JDK (_N_+1).
+
+The updates include changes to files holding meta-data about the
+release, files under the `src` directory for API and tooling updates,
+and incidental updates under the `test` directory.
+
+## Details and file updates
+
+As a matter of policy, there are a number of semantically distinct
+concepts which get incremented separately at the start of a new
+release:
+
+* Feature value of `Runtime.version()`
+* Highest source version modeled by `javax.lang.model.SourceVersion`
+* Highest class file format major version recognized by the platform
+* Highest `-source`/`-target`/`--release` argument recognized by
+ `javac` and related tools
+
+The expected file updates are listed below. Additional files may need
+to be updated for a particular release.
+
+### Meta-data files
+
+* `jcheck/conf`: update meta-data used by `jcheck` and the Skara tooling
+* `make/conf/version-numbers.conf`: update to meta-data used in the build
+
+### `src` files
+
+* `src/hotspot/share/classfile/classFileParser.cpp`: add a `#define`
+ for the new version
+* `src/java.base/share/classes/java/lang/classfile/ClassFile.java`:
+ add a constant for the new class file format version
+* `src/java.base/share/classes/java/lang/reflect/ClassFileFormatVersion.java`:
+ add an `enum` constant for the new class file format version
+* `src/java.compiler/share/classes/javax/lang/model/SourceVersion.java`:
+ add an `enum` constant for the new source version
+* `src/java.compiler/share/classes/javax/lang/model/util/*` visitors: Update
+ `@SupportedSourceVersion` annotations to latest value. Note this update
+ is done in lieu of introducing another set of visitors for each Java
+ SE release.
+* `src/jdk.compiler/share/classes/com/sun/tools/javac/code/Source.java`:
+ add an `enum` constant for the new source version internal to `javac`
+* `src/jdk.compiler/share/classes/com/sun/tools/javac/jvm/ClassFile.java`:
+ add an `enum` constant for the new class file format version internal to `javac`
+* `src/jdk.compiler/share/classes/com/sun/tools/javac/jvm/Target.java`:
+ add an `enum` constant for the new target version internal to `javac`
+* `src/jdk.compiler/share/classes/com/sun/tools/javac/processing/PrintingProcessor.java`
+ update printing processor to support the new source version
+* The symbol information for `--release` is stored as new text files in the
+ `src/jdk.compiler/share/data/symbols` directory, one file per
+ module. The README file in that directory contains directions on how
+ to create the files.
+
+### `test` files
+
+* `test/langtools/tools/javac/api/TestGetSourceVersions.java`: add new `SourceVersion` constant to test matrix.
+* `test/langtools/tools/javac/classfiles/ClassVersionChecker.java`: add new enum constant for the new class file version
+* `test/langtools/tools/javac/lib/JavacTestingAbstractProcessor.java`
+ update annotation processor extended by `javac` tests to cover the new source version
+* `test/langtools/tools/javac/preview/classReaderTest/Client.nopreview.out` and `test/langtools/tools/javac/preview/classReaderTest/Client.preview.out`: update expected messages for preview errors and warnings
+
diff --git a/make/Init.gmk b/make/Init.gmk
index 32624d7dee6..38959323628 100644
--- a/make/Init.gmk
+++ b/make/Init.gmk
@@ -110,7 +110,18 @@ reconfigure:
CUSTOM_CONFIG_DIR="$(CUSTOM_CONFIG_DIR)" \
$(RECONFIGURE_COMMAND) )
-.PHONY: print-modules print-targets print-tests print-configuration reconfigure
+# Create files that are needed to run most targets in Main.gmk
+create-make-helpers:
+ ( cd $(TOPDIR) && \
+ $(MAKE) $(MAKE_ARGS) -j 1 -f make/GenerateFindTests.gmk \
+ $(USER_MAKE_VARS) )
+ ( cd $(TOPDIR) && \
+ $(MAKE) $(MAKE_ARGS) -j 1 -f make/Main.gmk $(USER_MAKE_VARS) \
+ UPDATE_MODULE_DEPS=true NO_RECIPES=true \
+ create-main-targets-include )
+
+.PHONY: print-modules print-targets print-tests print-configuration \
+ reconfigure create-make-helpers
##############################################################################
# The main target. This will delegate all other targets into Main.gmk.
@@ -130,7 +141,7 @@ TARGET_DESCRIPTION := target$(if $(word 2, $(MAIN_TARGETS)),s) \
# variables are explicitly propagated using $(USER_MAKE_VARS).
main: MAKEOVERRIDES :=
-main: $(INIT_TARGETS)
+main: $(INIT_TARGETS) create-make-helpers
ifneq ($(SEQUENTIAL_TARGETS)$(PARALLEL_TARGETS), )
$(call RotateLogFiles)
$(ECHO) "Building $(TARGET_DESCRIPTION)" $(BUILD_LOG_PIPE_SIMPLE)
@@ -142,12 +153,7 @@ main: $(INIT_TARGETS)
$(SEQUENTIAL_TARGETS) )
# We might have cleaned away essential files, recreate them.
( cd $(TOPDIR) && \
- $(MAKE) $(MAKE_ARGS) -j 1 -f make/GenerateFindTests.gmk \
- $(USER_MAKE_VARS) )
- ( cd $(TOPDIR) && \
- $(MAKE) $(MAKE_ARGS) -j 1 -f make/Main.gmk $(USER_MAKE_VARS) \
- UPDATE_MODULE_DEPS=true NO_RECIPES=true \
- create-main-targets-include )
+ $(MAKE) $(MAKE_ARGS) -j 1 -f make/Init.gmk create-make-helpers )
endif
ifneq ($(PARALLEL_TARGETS), )
$(call PrepareFailureLogs)
diff --git a/make/Main.gmk b/make/Main.gmk
index 386bd226842..d0568509a4e 100644
--- a/make/Main.gmk
+++ b/make/Main.gmk
@@ -423,14 +423,6 @@ bootcycle-images:
ifneq ($(COMPILE_TYPE), cross)
$(call LogWarn, Boot cycle build step 2: Building a new JDK image using previously built image)
$(call MakeDir, $(OUTPUTDIR)/bootcycle-build)
- # We need to create essential files for the bootcycle spec dir
- ( cd $(TOPDIR) && \
- $(MAKE) $(MAKE_ARGS) -f make/GenerateFindTests.gmk \
- SPEC=$(BOOTCYCLE_SPEC))
- ( cd $(TOPDIR) && \
- $(MAKE) $(MAKE_ARGS) -f $(TOPDIR)/make/Main.gmk \
- SPEC=$(BOOTCYCLE_SPEC) UPDATE_MODULE_DEPS=true NO_RECIPES=true \
- create-main-targets-include )
+$(MAKE) $(MAKE_ARGS) -f $(TOPDIR)/make/Init.gmk PARALLEL_TARGETS=$(BOOTCYCLE_TARGET) \
LOG_PREFIX="[bootcycle] " JOBS= SPEC=$(BOOTCYCLE_SPEC) main
else
diff --git a/make/PreInit.gmk b/make/PreInit.gmk
index bce61ccde5f..3df44308dd9 100644
--- a/make/PreInit.gmk
+++ b/make/PreInit.gmk
@@ -50,7 +50,8 @@ include $(TOPDIR)/make/Global.gmk
# Targets provided by Init.gmk.
ALL_INIT_TARGETS := print-modules print-targets print-configuration \
- print-tests reconfigure pre-compare-build post-compare-build
+ print-tests reconfigure pre-compare-build post-compare-build \
+ create-make-helpers
# CALLED_TARGETS is the list of targets that the user provided,
# or "default" if unspecified.
diff --git a/make/autoconf/configure b/make/autoconf/configure
index 443a37bae77..98126d1558d 100644
--- a/make/autoconf/configure
+++ b/make/autoconf/configure
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -366,7 +366,7 @@ EOT
# Print additional help, e.g. a list of toolchains and JVM features.
# This must be done by the autoconf script.
- ( CONFIGURE_PRINT_ADDITIONAL_HELP=true . $generated_script PRINTF=printf )
+ ( CONFIGURE_PRINT_ADDITIONAL_HELP=true . $generated_script PRINTF=printf ECHO=echo )
cat <as_Load()->barrier_data() != 0);
effect(TEMP dst, KILL cr);
+ // The main load is a candidate to implement implicit null checks, as long as
+ // legitimize_address() does not require a preceding lea instruction to
+ // materialize the memory operand. The absence of a preceding lea instruction
+ // is guaranteed for immLoffset8 memory operands, because these do not lead to
+ // out-of-range offsets (see definition of immLoffset8). Fortunately,
+ // immLoffset8 memory operands are the most common ones in practice.
+ ins_is_late_expanded_null_check_candidate(opnd_array(1)->opcode() == INDOFFL8);
ins_cost(4 * INSN_COST);
@@ -117,7 +124,11 @@ instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
// Fix up any out-of-range offsets.
assert_different_registers(rscratch2, as_Register($mem$$base));
assert_different_registers(rscratch2, $dst$$Register);
- ref_addr = __ legitimize_address(ref_addr, 8, rscratch2);
+ int size = 8;
+ assert(!this->is_late_expanded_null_check_candidate() ||
+ !MacroAssembler::legitimize_address_requires_lea(ref_addr, size),
+ "an instruction that can be used for implicit null checking should emit the candidate memory access first");
+ ref_addr = __ legitimize_address(ref_addr, size, rscratch2);
}
__ ldr($dst$$Register, ref_addr);
z_load_barrier(masm, this, ref_addr, $dst$$Register, rscratch1);
diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
index 632bba728a1..b316103d656 100644
--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2019, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -95,6 +95,8 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Use simplest and shortest implementation for array equals") \
product(bool, UseSIMDForBigIntegerShiftIntrinsics, true, \
"Use SIMD instructions for left/right shift of BigInteger") \
+ product(bool, UseSIMDForSHA3Intrinsic, true, \
+ "Use SIMD SHA3 instructions for SHA3 intrinsic") \
product(bool, AvoidUnalignedAccesses, false, \
"Avoid generating unaligned memory accesses") \
product(bool, UseLSE, false, \
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index 32506c49cfa..f5f0f630c0c 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -129,16 +129,21 @@ class MacroAssembler: public Assembler {
a.lea(this, r);
}
+ // Whether materializing the given address for a LDR/STR requires an
+ // additional lea instruction.
+ static bool legitimize_address_requires_lea(const Address &a, int size) {
+ return a.getMode() == Address::base_plus_offset &&
+ !Address::offset_ok_for_immed(a.offset(), exact_log2(size));
+ }
+
/* Sometimes we get misaligned loads and stores, usually from Unsafe
accesses, and these can exceed the offset range. */
Address legitimize_address(const Address &a, int size, Register scratch) {
- if (a.getMode() == Address::base_plus_offset) {
- if (! Address::offset_ok_for_immed(a.offset(), exact_log2(size))) {
- block_comment("legitimize_address {");
- lea(scratch, a);
- block_comment("} legitimize_address");
- return Address(scratch);
- }
+ if (legitimize_address_requires_lea(a, size)) {
+ block_comment("legitimize_address {");
+ lea(scratch, a);
+ block_comment("} legitimize_address");
+ return Address(scratch);
}
return a;
}
@@ -323,6 +328,27 @@ class MacroAssembler: public Assembler {
extr(Rd, Rn, Rn, imm);
}
+ inline void rolw(Register Rd, Register Rn, unsigned imm) {
+ extrw(Rd, Rn, Rn, (32 - imm));
+ }
+
+ inline void rol(Register Rd, Register Rn, unsigned imm) {
+ extr(Rd, Rn, Rn, (64 - imm));
+ }
+
+ using Assembler::rax1;
+ using Assembler::eor3;
+
+ inline void rax1(Register Rd, Register Rn, Register Rm) {
+ eor(Rd, Rn, Rm, ROR, 63); // Rd = Rn ^ rol(Rm, 1)
+ }
+
+ inline void eor3(Register Rd, Register Rn, Register Rm, Register Rk) {
+ assert(Rd != Rn, "Use tmp register");
+ eor(Rd, Rm, Rk);
+ eor(Rd, Rd, Rn);
+ }
+
inline void sxtbw(Register Rd, Register Rn) {
sbfmw(Rd, Rn, 0, 7);
}
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index 1b6f6d489f3..a0d1e22ff96 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -7081,6 +7081,366 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
+ void bcax5(Register a0, Register a1, Register a2, Register a3, Register a4,
+ Register tmp0, Register tmp1, Register tmp2) {
+ __ bic(tmp0, a2, a1); // for a0
+ __ bic(tmp1, a3, a2); // for a1
+ __ bic(tmp2, a4, a3); // for a2
+ __ eor(a2, a2, tmp2);
+ __ bic(tmp2, a0, a4); // for a3
+ __ eor(a3, a3, tmp2);
+ __ bic(tmp2, a1, a0); // for a4
+ __ eor(a0, a0, tmp0);
+ __ eor(a1, a1, tmp1);
+ __ eor(a4, a4, tmp2);
+ }
+
+ void keccak_round_gpr(bool can_use_fp, bool can_use_r18, Register rc,
+ Register a0, Register a1, Register a2, Register a3, Register a4,
+ Register a5, Register a6, Register a7, Register a8, Register a9,
+ Register a10, Register a11, Register a12, Register a13, Register a14,
+ Register a15, Register a16, Register a17, Register a18, Register a19,
+ Register a20, Register a21, Register a22, Register a23, Register a24,
+ Register tmp0, Register tmp1, Register tmp2) {
+ __ eor3(tmp1, a4, a9, a14);
+ __ eor3(tmp0, tmp1, a19, a24); // tmp0 = a4^a9^a14^a19^a24 = c4
+ __ eor3(tmp2, a1, a6, a11);
+ __ eor3(tmp1, tmp2, a16, a21); // tmp1 = a1^a6^a11^a16^a21 = c1
+ __ rax1(tmp2, tmp0, tmp1); // d0
+ {
+
+ Register tmp3, tmp4;
+ if (can_use_fp && can_use_r18) {
+ tmp3 = rfp;
+ tmp4 = r18_tls;
+ } else {
+ tmp3 = a4;
+ tmp4 = a9;
+ __ stp(tmp3, tmp4, __ pre(sp, -16));
+ }
+
+ __ eor3(tmp3, a0, a5, a10);
+ __ eor3(tmp4, tmp3, a15, a20); // tmp4 = a0^a5^a10^a15^a20 = c0
+ __ eor(a0, a0, tmp2);
+ __ eor(a5, a5, tmp2);
+ __ eor(a10, a10, tmp2);
+ __ eor(a15, a15, tmp2);
+ __ eor(a20, a20, tmp2); // d0(tmp2)
+ __ eor3(tmp3, a2, a7, a12);
+ __ eor3(tmp2, tmp3, a17, a22); // tmp2 = a2^a7^a12^a17^a22 = c2
+ __ rax1(tmp3, tmp4, tmp2); // d1
+ __ eor(a1, a1, tmp3);
+ __ eor(a6, a6, tmp3);
+ __ eor(a11, a11, tmp3);
+ __ eor(a16, a16, tmp3);
+ __ eor(a21, a21, tmp3); // d1(tmp3)
+ __ rax1(tmp3, tmp2, tmp0); // d3
+ __ eor3(tmp2, a3, a8, a13);
+ __ eor3(tmp0, tmp2, a18, a23); // tmp0 = a3^a8^a13^a18^a23 = c3
+ __ eor(a3, a3, tmp3);
+ __ eor(a8, a8, tmp3);
+ __ eor(a13, a13, tmp3);
+ __ eor(a18, a18, tmp3);
+ __ eor(a23, a23, tmp3);
+ __ rax1(tmp2, tmp1, tmp0); // d2
+ __ eor(a2, a2, tmp2);
+ __ eor(a7, a7, tmp2);
+ __ eor(a12, a12, tmp2);
+ __ rax1(tmp0, tmp0, tmp4); // d4
+ if (!can_use_fp || !can_use_r18) {
+ __ ldp(tmp3, tmp4, __ post(sp, 16));
+ }
+ __ eor(a17, a17, tmp2);
+ __ eor(a22, a22, tmp2);
+ __ eor(a4, a4, tmp0);
+ __ eor(a9, a9, tmp0);
+ __ eor(a14, a14, tmp0);
+ __ eor(a19, a19, tmp0);
+ __ eor(a24, a24, tmp0);
+ }
+
+ __ rol(tmp0, a10, 3);
+ __ rol(a10, a1, 1);
+ __ rol(a1, a6, 44);
+ __ rol(a6, a9, 20);
+ __ rol(a9, a22, 61);
+ __ rol(a22, a14, 39);
+ __ rol(a14, a20, 18);
+ __ rol(a20, a2, 62);
+ __ rol(a2, a12, 43);
+ __ rol(a12, a13, 25);
+ __ rol(a13, a19, 8) ;
+ __ rol(a19, a23, 56);
+ __ rol(a23, a15, 41);
+ __ rol(a15, a4, 27);
+ __ rol(a4, a24, 14);
+ __ rol(a24, a21, 2);
+ __ rol(a21, a8, 55);
+ __ rol(a8, a16, 45);
+ __ rol(a16, a5, 36);
+ __ rol(a5, a3, 28);
+ __ rol(a3, a18, 21);
+ __ rol(a18, a17, 15);
+ __ rol(a17, a11, 10);
+ __ rol(a11, a7, 6);
+ __ mov(a7, tmp0);
+
+ bcax5(a0, a1, a2, a3, a4, tmp0, tmp1, tmp2);
+ bcax5(a5, a6, a7, a8, a9, tmp0, tmp1, tmp2);
+ bcax5(a10, a11, a12, a13, a14, tmp0, tmp1, tmp2);
+ bcax5(a15, a16, a17, a18, a19, tmp0, tmp1, tmp2);
+ bcax5(a20, a21, a22, a23, a24, tmp0, tmp1, tmp2);
+
+ __ ldr(tmp1, __ post(rc, 8));
+ __ eor(a0, a0, tmp1);
+
+ }
+
+ // Arguments:
+ //
+ // Inputs:
+ // c_rarg0 - byte[] source+offset
+ // c_rarg1 - byte[] SHA.state
+ // c_rarg2 - int block_size
+ // c_rarg3 - int offset
+ // c_rarg4 - int limit
+ //
+ address generate_sha3_implCompress_gpr(StubGenStubId stub_id) {
+ bool multi_block;
+ switch (stub_id) {
+ case sha3_implCompress_id:
+ multi_block = false;
+ break;
+ case sha3_implCompressMB_id:
+ multi_block = true;
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+
+ static const uint64_t round_consts[24] = {
+ 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
+ 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
+ 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
+ 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
+ 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
+ 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
+ 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
+ 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
+ };
+
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
+ address start = __ pc();
+
+ Register buf = c_rarg0;
+ Register state = c_rarg1;
+ Register block_size = c_rarg2;
+ Register ofs = c_rarg3;
+ Register limit = c_rarg4;
+
+ // use r3.r17,r19..r28 to keep a0..a24.
+ // a0..a24 are respective locals from SHA3.java
+ Register a0 = r25,
+ a1 = r26,
+ a2 = r27,
+ a3 = r3,
+ a4 = r4,
+ a5 = r5,
+ a6 = r6,
+ a7 = r7,
+ a8 = rscratch1, // r8
+ a9 = rscratch2, // r9
+ a10 = r10,
+ a11 = r11,
+ a12 = r12,
+ a13 = r13,
+ a14 = r14,
+ a15 = r15,
+ a16 = r16,
+ a17 = r17,
+ a18 = r28,
+ a19 = r19,
+ a20 = r20,
+ a21 = r21,
+ a22 = r22,
+ a23 = r23,
+ a24 = r24;
+
+ Register tmp0 = block_size, tmp1 = buf, tmp2 = state, tmp3 = r30;
+
+ Label sha3_loop, rounds24_preloop, loop_body;
+ Label sha3_512_or_sha3_384, shake128;
+
+ bool can_use_r18 = false;
+#ifndef R18_RESERVED
+ can_use_r18 = true;
+#endif
+ bool can_use_fp = !PreserveFramePointer;
+
+ __ enter();
+
+ // save almost all yet unsaved gpr registers on stack
+ __ str(block_size, __ pre(sp, -128));
+ if (multi_block) {
+ __ stpw(ofs, limit, Address(sp, 8));
+ }
+ // 8 bytes at sp+16 will be used to keep buf
+ __ stp(r19, r20, Address(sp, 32));
+ __ stp(r21, r22, Address(sp, 48));
+ __ stp(r23, r24, Address(sp, 64));
+ __ stp(r25, r26, Address(sp, 80));
+ __ stp(r27, r28, Address(sp, 96));
+ if (can_use_r18 && can_use_fp) {
+ __ stp(r18_tls, state, Address(sp, 112));
+ } else {
+ __ str(state, Address(sp, 112));
+ }
+
+ // begin sha3 calculations: loading a0..a24 from state arrary
+ __ ldp(a0, a1, state);
+ __ ldp(a2, a3, Address(state, 16));
+ __ ldp(a4, a5, Address(state, 32));
+ __ ldp(a6, a7, Address(state, 48));
+ __ ldp(a8, a9, Address(state, 64));
+ __ ldp(a10, a11, Address(state, 80));
+ __ ldp(a12, a13, Address(state, 96));
+ __ ldp(a14, a15, Address(state, 112));
+ __ ldp(a16, a17, Address(state, 128));
+ __ ldp(a18, a19, Address(state, 144));
+ __ ldp(a20, a21, Address(state, 160));
+ __ ldp(a22, a23, Address(state, 176));
+ __ ldr(a24, Address(state, 192));
+
+ __ BIND(sha3_loop);
+
+ // load input
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a0, a0, tmp3);
+ __ eor(a1, a1, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a2, a2, tmp3);
+ __ eor(a3, a3, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a4, a4, tmp3);
+ __ eor(a5, a5, tmp2);
+ __ ldr(tmp3, __ post(buf, 8));
+ __ eor(a6, a6, tmp3);
+
+ // block_size == 72, SHA3-512; block_size == 104, SHA3-384
+ __ tbz(block_size, 7, sha3_512_or_sha3_384);
+
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a7, a7, tmp3);
+ __ eor(a8, a8, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a9, a9, tmp3);
+ __ eor(a10, a10, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a11, a11, tmp3);
+ __ eor(a12, a12, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a13, a13, tmp3);
+ __ eor(a14, a14, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a15, a15, tmp3);
+ __ eor(a16, a16, tmp2);
+
+ // block_size == 136, bit4 == 0 and bit5 == 0, SHA3-256 or SHAKE256
+ __ andw(tmp2, block_size, 48);
+ __ cbzw(tmp2, rounds24_preloop);
+ __ tbnz(block_size, 5, shake128);
+ // block_size == 144, bit5 == 0, SHA3-244
+ __ ldr(tmp3, __ post(buf, 8));
+ __ eor(a17, a17, tmp3);
+ __ b(rounds24_preloop);
+
+ __ BIND(shake128);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a17, a17, tmp3);
+ __ eor(a18, a18, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a19, a19, tmp3);
+ __ eor(a20, a20, tmp2);
+ __ b(rounds24_preloop); // block_size == 168, SHAKE128
+
+ __ BIND(sha3_512_or_sha3_384);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a7, a7, tmp3);
+ __ eor(a8, a8, tmp2);
+ __ tbz(block_size, 5, rounds24_preloop); // SHA3-512
+
+ // SHA3-384
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a9, a9, tmp3);
+ __ eor(a10, a10, tmp2);
+ __ ldp(tmp3, tmp2, __ post(buf, 16));
+ __ eor(a11, a11, tmp3);
+ __ eor(a12, a12, tmp2);
+
+ __ BIND(rounds24_preloop);
+ __ fmovs(v0, 24.0); // float loop counter,
+ __ fmovs(v1, 1.0); // exact representation
+
+ __ str(buf, Address(sp, 16));
+ __ lea(tmp3, ExternalAddress((address) round_consts));
+
+ __ BIND(loop_body);
+ keccak_round_gpr(can_use_fp, can_use_r18, tmp3,
+ a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12,
+ a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24,
+ tmp0, tmp1, tmp2);
+ __ fsubs(v0, v0, v1);
+ __ fcmps(v0, 0.0);
+ __ br(__ NE, loop_body);
+
+ if (multi_block) {
+ __ ldrw(block_size, sp); // block_size
+ __ ldpw(tmp2, tmp1, Address(sp, 8)); // offset, limit
+ __ addw(tmp2, tmp2, block_size);
+ __ cmpw(tmp2, tmp1);
+ __ strw(tmp2, Address(sp, 8)); // store offset in case we're jumping
+ __ ldr(buf, Address(sp, 16)); // restore buf in case we're jumping
+ __ br(Assembler::LE, sha3_loop);
+ __ movw(c_rarg0, tmp2); // return offset
+ }
+ if (can_use_fp && can_use_r18) {
+ __ ldp(r18_tls, state, Address(sp, 112));
+ } else {
+ __ ldr(state, Address(sp, 112));
+ }
+ // save calculated sha3 state
+ __ stp(a0, a1, Address(state));
+ __ stp(a2, a3, Address(state, 16));
+ __ stp(a4, a5, Address(state, 32));
+ __ stp(a6, a7, Address(state, 48));
+ __ stp(a8, a9, Address(state, 64));
+ __ stp(a10, a11, Address(state, 80));
+ __ stp(a12, a13, Address(state, 96));
+ __ stp(a14, a15, Address(state, 112));
+ __ stp(a16, a17, Address(state, 128));
+ __ stp(a18, a19, Address(state, 144));
+ __ stp(a20, a21, Address(state, 160));
+ __ stp(a22, a23, Address(state, 176));
+ __ str(a24, Address(state, 192));
+
+ // restore required registers from stack
+ __ ldp(r19, r20, Address(sp, 32));
+ __ ldp(r21, r22, Address(sp, 48));
+ __ ldp(r23, r24, Address(sp, 64));
+ __ ldp(r25, r26, Address(sp, 80));
+ __ ldp(r27, r28, Address(sp, 96));
+ if (can_use_fp && can_use_r18) {
+ __ add(rfp, sp, 128); // leave() will copy rfp to sp below
+ } // else no need to recalculate rfp, since it wasn't changed
+
+ __ leave();
+
+ __ ret(lr);
+
+ return start;
+ }
+
/**
* Arguments:
*
@@ -11512,9 +11872,15 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubGenStubId::sha512_implCompressMB_id);
}
if (UseSHA3Intrinsics) {
- StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubGenStubId::sha3_implCompress_id);
+
StubRoutines::_double_keccak = generate_double_keccak();
- StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubGenStubId::sha3_implCompressMB_id);
+ if (UseSIMDForSHA3Intrinsic) {
+ StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubGenStubId::sha3_implCompress_id);
+ StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubGenStubId::sha3_implCompressMB_id);
+ } else {
+ StubRoutines::_sha3_implCompress = generate_sha3_implCompress_gpr(StubGenStubId::sha3_implCompress_id);
+ StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress_gpr(StubGenStubId::sha3_implCompressMB_id);
+ }
}
if (UsePoly1305Intrinsics) {
diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
index 0ed064f48ba..710970d1ea2 100644
--- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
@@ -1893,6 +1893,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
Interpreter::_remove_activation_preserving_args_entry = __ pc();
__ empty_expression_stack();
+ __ restore_bcp(); // We could have returned from deoptimizing this frame, so restore rbcp.
// Set the popframe_processing bit in pending_popframe_condition
// indicating that we are currently handling popframe, so that
// call_VMs that may happen later do not trigger new popframe
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index 6ed7a6be585..941cb254532 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -379,7 +379,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
}
}
- } else if (UseSHA3Intrinsics) {
+ } else if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic) {
warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
diff --git a/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.cpp
index 89417c4691d..20d96f6e937 100644
--- a/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.cpp
@@ -93,10 +93,15 @@ static size_t probe_valid_max_address_bit() {
size_t ZPlatformAddressOffsetBits() {
static const size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
+#ifdef ADDRESS_SANITIZER
+ // The max supported value is 44 because of other internal data structures.
+ return MIN2(valid_max_address_offset_bits, (size_t)44);
+#else
const size_t min_address_offset_bits = max_address_offset_bits - 2;
const size_t address_offset = ZGlobalsPointers::min_address_offset_request();
const size_t address_offset_bits = log2i_exact(address_offset);
return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
+#endif
}
size_t ZPlatformAddressHeapBaseShift() {
diff --git a/src/hotspot/cpu/ppc/gc/z/z_ppc.ad b/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
index 65fb206ad7e..0872932ccb2 100644
--- a/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
+++ b/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
@@ -141,6 +141,7 @@ instruct zLoadP(iRegPdst dst, memoryAlg4 mem, flagsRegCR0 cr0)
%{
match(Set dst (LoadP mem));
effect(TEMP_DEF dst, KILL cr0);
+ ins_is_late_expanded_null_check_candidate(true);
ins_cost(MEMORY_REF_COST);
predicate((UseZGC && n->as_Load()->barrier_data() != 0)
@@ -160,6 +161,7 @@ instruct zLoadP_acq(iRegPdst dst, memoryAlg4 mem, flagsRegCR0 cr0)
%{
match(Set dst (LoadP mem));
effect(TEMP_DEF dst, KILL cr0);
+ ins_is_late_expanded_null_check_candidate(true);
ins_cost(3 * MEMORY_REF_COST);
// Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index 128e566d0f3..d8e00cfef89 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -4036,6 +4036,10 @@ ins_attrib ins_field_cbuf_insts_offset(-1);
ins_attrib ins_field_load_ic_hi_node(0);
ins_attrib ins_field_load_ic_node(0);
+// Whether this node is expanded during code emission into a sequence of
+// instructions and the first instruction can perform an implicit null check.
+ins_attrib ins_is_late_expanded_null_check_candidate(false);
+
//----------OPERANDS-----------------------------------------------------------
// Operand definitions must precede instruction definitions for correct
// parsing in the ADLC because operands constitute user defined types
diff --git a/src/hotspot/cpu/ppc/register_ppc.hpp b/src/hotspot/cpu/ppc/register_ppc.hpp
index 2b5b25f449e..b7949750dcc 100644
--- a/src/hotspot/cpu/ppc/register_ppc.hpp
+++ b/src/hotspot/cpu/ppc/register_ppc.hpp
@@ -523,7 +523,7 @@ constexpr FloatRegister F11_ARG11 = F11; // volatile
constexpr FloatRegister F12_ARG12 = F12; // volatile
constexpr FloatRegister F13_ARG13 = F13; // volatile
-// Register declarations to be used in frame manager assembly code.
+// Register declarations to be used in template interpreter assembly code.
// Use only non-volatile registers in order to keep values across C-calls.
constexpr Register R14_bcp = R14;
constexpr Register R15_esp = R15; // slot below top of expression stack for ld/st with update
@@ -533,7 +533,7 @@ constexpr Register R17_tos = R17; // The interpreter's top of (expres
constexpr Register R18_locals = R18; // address of first param slot (receiver).
constexpr Register R19_method = R19; // address of current method
-// Temporary registers to be used within frame manager. We can use
+// Temporary registers to be used within template interpreter. We can use
// the non-volatiles because the call stub has saved them.
// Use only non-volatile registers in order to keep values across C-calls.
constexpr Register R21_tmp1 = R21;
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index a7e759d770b..4ec2483b267 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -2935,7 +2935,7 @@ static void push_skeleton_frames(MacroAssembler* masm, bool deopt,
__ cmpdi(CR0, number_of_frames_reg, 0);
__ bne(CR0, loop);
- // Get the return address pointing into the frame manager.
+ // Get the return address pointing into the template interpreter.
__ ld(R0, 0, pcs_reg);
// Store it in the top interpreter frame.
__ std(R0, _abi0(lr), R1_SP);
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index 7fc807bb9ce..2624131033c 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -86,7 +86,7 @@ class StubGenerator: public StubCodeGenerator {
// R10 - thread : Thread*
//
address generate_call_stub(address& return_address) {
- // Setup a new c frame, copy java arguments, call frame manager or
+ // Setup a new c frame, copy java arguments, call template interpreter or
// native_entry, and process result.
StubGenStubId stub_id = StubGenStubId::call_stub_id;
@@ -215,11 +215,10 @@ class StubGenerator: public StubCodeGenerator {
}
{
- BLOCK_COMMENT("Call frame manager or native entry.");
- // Call frame manager or native entry.
+ BLOCK_COMMENT("Call template interpreter or native entry.");
assert_different_registers(r_arg_entry, r_top_of_arguments_addr, r_arg_method, r_arg_thread);
- // Register state on entry to frame manager / native entry:
+ // Register state on entry to template interpreter / native entry:
//
// tos - intptr_t* sender tos (prepushed) Lesp = (SP) + copied_arguments_offset - 8
// R19_method - Method
@@ -242,7 +241,7 @@ class StubGenerator: public StubCodeGenerator {
// Set R15_prev_state to 0 for simplifying checks in callee.
__ load_const_optimized(R25_templateTableBase, (address)Interpreter::dispatch_table((TosState)0), R0);
- // Stack on entry to frame manager / native entry:
+ // Stack on entry to template interpreter / native entry:
//
// F0 [TOP_IJAVA_FRAME_ABI]
// alignment (optional)
@@ -262,7 +261,7 @@ class StubGenerator: public StubCodeGenerator {
__ mr(R21_sender_SP, R1_SP);
// Do a light-weight C-call here, r_arg_entry holds the address
- // of the interpreter entry point (frame manager or native entry)
+ // of the interpreter entry point (template interpreter or native entry)
// and save runtime-value of LR in return_address.
assert(r_arg_entry != tos && r_arg_entry != R19_method && r_arg_entry != R16_thread,
"trashed r_arg_entry");
@@ -270,11 +269,10 @@ class StubGenerator: public StubCodeGenerator {
}
{
- BLOCK_COMMENT("Returned from frame manager or native entry.");
- // Returned from frame manager or native entry.
+ BLOCK_COMMENT("Returned from template interpreter or native entry.");
// Now pop frame, process result, and return to caller.
- // Stack on exit from frame manager / native entry:
+ // Stack on exit from template interpreter / native entry:
//
// F0 [ABI]
// ...
@@ -295,7 +293,7 @@ class StubGenerator: public StubCodeGenerator {
Register r_cr = R12_scratch2;
// Reload some volatile registers which we've spilled before the call
- // to frame manager / native entry.
+ // to template interpreter / native entry.
// Access all locals via frame pointer, because we know nothing about
// the topmost frame's size.
__ ld(r_entryframe_fp, _abi0(callers_sp), R1_SP); // restore after call
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index 3cdc9b70d52..3317ccc3b53 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -961,81 +961,239 @@ protected:
#undef INSN
-enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
+ enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
-#define INSN(NAME, op, funct3, funct7) \
- void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { \
- unsigned insn = 0; \
- patch((address)&insn, 6, 0, op); \
- patch((address)&insn, 14, 12, funct3); \
- patch_reg((address)&insn, 7, Rd); \
- patch_reg((address)&insn, 15, Rs1); \
- patch_reg((address)&insn, 20, Rs2); \
- patch((address)&insn, 31, 27, funct7); \
- patch((address)&insn, 26, 25, memory_order); \
- emit(insn); \
+ private:
+
+ enum AmoWidthFunct3 : uint8_t {
+ AMO_WIDTH_BYTE = 0b000, // Zabha extension
+ AMO_WIDTH_HALFWORD = 0b001, // Zabha extension
+ AMO_WIDTH_WORD = 0b010,
+ AMO_WIDTH_DOUBLEWORD = 0b011,
+ AMO_WIDTH_QUADWORD = 0b100,
+ // 0b101 to 0b111 are reserved
+ };
+
+ enum AmoOperationFunct5 : uint8_t {
+ AMO_ADD = 0b00000,
+ AMO_SWAP = 0b00001,
+ AMO_LR = 0b00010,
+ AMO_SC = 0b00011,
+ AMO_XOR = 0b00100,
+ AMO_OR = 0b01000,
+ AMO_AND = 0b01100,
+ AMO_MIN = 0b10000,
+ AMO_MAX = 0b10100,
+ AMO_MINU = 0b11000,
+ AMO_MAXU = 0b11100,
+ AMO_CAS = 0b00101 // Zacas
+ };
+
+ static constexpr uint32_t OP_AMO_MAJOR = 0b0101111;
+
+ template
+ void amo_base(Register Rd, Register Rs1, uint8_t Rs2, Aqrl memory_order = aqrl) {
+ assert(width > AMO_WIDTH_HALFWORD || UseZabha, "Must be");
+ assert(funct5 != AMO_CAS || UseZacas, "Must be");
+ unsigned insn = 0;
+ patch((address)&insn, 6, 0, OP_AMO_MAJOR);
+ patch_reg((address)&insn, 7, Rd);
+ patch((address)&insn, 14, 12, width);
+ patch_reg((address)&insn, 15, Rs1);
+ patch((address)&insn, 24, 20, Rs2);
+ patch((address)&insn, 26, 25, memory_order);
+ patch((address)&insn, 31, 27, funct5);
+ emit(insn);
}
- INSN(amoswap_w, 0b0101111, 0b010, 0b00001);
- INSN(amoadd_w, 0b0101111, 0b010, 0b00000);
- INSN(amoxor_w, 0b0101111, 0b010, 0b00100);
- INSN(amoand_w, 0b0101111, 0b010, 0b01100);
- INSN(amoor_w, 0b0101111, 0b010, 0b01000);
- INSN(amomin_w, 0b0101111, 0b010, 0b10000);
- INSN(amomax_w, 0b0101111, 0b010, 0b10100);
- INSN(amominu_w, 0b0101111, 0b010, 0b11000);
- INSN(amomaxu_w, 0b0101111, 0b010, 0b11100);
- INSN(amoswap_d, 0b0101111, 0b011, 0b00001);
- INSN(amoadd_d, 0b0101111, 0b011, 0b00000);
- INSN(amoxor_d, 0b0101111, 0b011, 0b00100);
- INSN(amoand_d, 0b0101111, 0b011, 0b01100);
- INSN(amoor_d, 0b0101111, 0b011, 0b01000);
- INSN(amomin_d, 0b0101111, 0b011, 0b10000);
- INSN(amomax_d , 0b0101111, 0b011, 0b10100);
- INSN(amominu_d, 0b0101111, 0b011, 0b11000);
- INSN(amomaxu_d, 0b0101111, 0b011, 0b11100);
- INSN(amocas_w, 0b0101111, 0b010, 0b00101);
- INSN(amocas_d, 0b0101111, 0b011, 0b00101);
-#undef INSN
-
-enum operand_size { int8, int16, int32, uint32, int64 };
-
-#define INSN(NAME, op, funct3, funct7) \
- void NAME(Register Rd, Register Rs1, Aqrl memory_order = relaxed) { \
- unsigned insn = 0; \
- uint32_t val = memory_order & 0x3; \
- patch((address)&insn, 6, 0, op); \
- patch((address)&insn, 14, 12, funct3); \
- patch_reg((address)&insn, 7, Rd); \
- patch_reg((address)&insn, 15, Rs1); \
- patch((address)&insn, 25, 20, 0b00000); \
- patch((address)&insn, 31, 27, funct7); \
- patch((address)&insn, 26, 25, val); \
- emit(insn); \
+ template
+ void amo_base(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2->raw_encoding(), memory_order);
}
- INSN(lr_w, 0b0101111, 0b010, 0b00010);
- INSN(lr_d, 0b0101111, 0b011, 0b00010);
+ public:
-#undef INSN
-
-#define INSN(NAME, op, funct3, funct7) \
- void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = relaxed) { \
- unsigned insn = 0; \
- uint32_t val = memory_order & 0x3; \
- patch((address)&insn, 6, 0, op); \
- patch((address)&insn, 14, 12, funct3); \
- patch_reg((address)&insn, 7, Rd); \
- patch_reg((address)&insn, 15, Rs2); \
- patch_reg((address)&insn, 20, Rs1); \
- patch((address)&insn, 31, 27, funct7); \
- patch((address)&insn, 26, 25, val); \
- emit(insn); \
+ void amoadd_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
}
- INSN(sc_w, 0b0101111, 0b010, 0b00011);
- INSN(sc_d, 0b0101111, 0b011, 0b00011);
-#undef INSN
+ void amoadd_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoadd_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoadd_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ protected:
+
+ void lr_w(Register Rd, Register Rs1, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, 0, memory_order);
+ }
+
+ void lr_d(Register Rd, Register Rs1, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, 0, memory_order);
+ }
+
+ void sc_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void sc_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base(Rd, Rs1, Rs2, memory_order);
+ }
+
+ public:
+
+ enum operand_size { int8, int16, int32, uint32, int64 };
// Immediate Instruction
#define INSN(NAME, op, funct3) \
diff --git a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
index fd9a1d43afc..e3847019bb3 100644
--- a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
+++ b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
@@ -96,6 +96,7 @@ instruct zLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp, rFlagsReg cr)
match(Set dst (LoadP mem));
predicate(UseZGC && n->as_Load()->barrier_data() != 0);
effect(TEMP dst, TEMP tmp, KILL cr);
+ ins_is_late_expanded_null_check_candidate(true);
ins_cost(4 * DEFAULT_COST);
diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp
index 3ef084d30fc..d67e05bbb6d 100644
--- a/src/hotspot/cpu/riscv/globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globals_riscv.hpp
@@ -107,6 +107,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseZfh, false, DIAGNOSTIC, "Use Zfh instructions") \
product(bool, UseZfhmin, false, DIAGNOSTIC, "Use Zfhmin instructions") \
product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \
+ product(bool, UseZabha, false, EXPERIMENTAL, "Use UseZabha instructions") \
product(bool, UseZcb, false, EXPERIMENTAL, "Use Zcb instructions") \
product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions") \
product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 2a01a6d209d..fae34a9c770 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -955,47 +955,29 @@ void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in,
void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
- int constant,
- bool decrement) {
- increment_mdp_data_at(mdp_in, noreg, constant, decrement);
+ int constant) {
+ increment_mdp_data_at(mdp_in, noreg, constant);
}
void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
- Register reg,
- int constant,
- bool decrement) {
+ Register index,
+ int constant) {
assert(ProfileInterpreter, "must be profiling interpreter");
- // %%% this does 64bit counters at best it is wasting space
- // at worst it is a rare bug when counters overflow
- assert_different_registers(t1, t0, mdp_in, reg);
+ assert_different_registers(t1, t0, mdp_in, index);
Address addr1(mdp_in, constant);
Address addr2(t1, 0);
Address &addr = addr1;
- if (reg != noreg) {
+ if (index != noreg) {
la(t1, addr1);
- add(t1, t1, reg);
+ add(t1, t1, index);
addr = addr2;
}
- if (decrement) {
- ld(t0, addr);
- subi(t0, t0, DataLayout::counter_increment);
- Label L;
- bltz(t0, L); // skip store if counter underflow
- sd(t0, addr);
- bind(L);
- } else {
- assert(DataLayout::counter_increment == 1,
- "flow-free idiom only works with 1");
- ld(t0, addr);
- addi(t0, t0, DataLayout::counter_increment);
- Label L;
- blez(t0, L); // skip store if counter overflow
- sd(t0, addr);
- bind(L);
- }
+ ld(t0, addr);
+ addi(t0, t0, DataLayout::counter_increment);
+ sd(t0, addr);
}
void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in,
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
index 7a3e6764aaa..891db16b243 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
@@ -233,11 +233,8 @@ class InterpreterMacroAssembler: public MacroAssembler {
void verify_method_data_pointer();
void set_mdp_data_at(Register mdp_in, int constant, Register value);
- void increment_mdp_data_at(Address data, bool decrement = false);
- void increment_mdp_data_at(Register mdp_in, int constant,
- bool decrement = false);
- void increment_mdp_data_at(Register mdp_in, Register reg, int constant,
- bool decrement = false);
+ void increment_mdp_data_at(Register mdp_in, int constant);
+ void increment_mdp_data_at(Register mdp_in, Register index, int constant);
void increment_mask_and_jump(Address counter_addr,
int increment, Address mask,
Register tmp1, Register tmp2,
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index bc6bf88d0ad..c755d9ae23d 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -3798,7 +3798,7 @@ void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register o
void MacroAssembler::load_reserved(Register dst,
Register addr,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire) {
switch (size) {
case int64:
@@ -3819,15 +3819,15 @@ void MacroAssembler::load_reserved(Register dst,
void MacroAssembler::store_conditional(Register dst,
Register new_val,
Register addr,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl release) {
switch (size) {
case int64:
- sc_d(dst, new_val, addr, release);
+ sc_d(dst, addr, new_val, release);
break;
case int32:
case uint32:
- sc_w(dst, new_val, addr, release);
+ sc_w(dst, addr, new_val, release);
break;
default:
ShouldNotReachHere();
@@ -3836,7 +3836,7 @@ void MacroAssembler::store_conditional(Register dst,
void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Register shift, Register mask, Register aligned_addr) {
assert(size == int8 || size == int16, "unsupported operand size");
@@ -3866,10 +3866,11 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
// which are forced to work with 4-byte aligned address.
void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool,
Register tmp1, Register tmp2, Register tmp3) {
+ assert(!(UseZacas && UseZabha), "Use amocas");
assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);
Register scratch0 = t0, aligned_addr = t1;
@@ -3902,13 +3903,13 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
notr(scratch1, mask);
bind(retry);
- lr_w(result, aligned_addr, acquire);
+ load_reserved(result, aligned_addr, operand_size::int32, acquire);
andr(scratch0, result, mask);
bne(scratch0, expected, fail);
andr(scratch0, result, scratch1); // scratch1 is ~mask
orr(scratch0, scratch0, new_val);
- sc_w(scratch0, scratch0, aligned_addr, release);
+ store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release);
bnez(scratch0, retry);
}
@@ -3940,10 +3941,11 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
// failed.
void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result,
Register tmp1, Register tmp2, Register tmp3) {
+ assert(!(UseZacas && UseZabha), "Use amocas");
assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);
Register scratch0 = t0, aligned_addr = t1;
@@ -3974,13 +3976,13 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
} else {
notr(scratch1, mask);
- lr_w(result, aligned_addr, acquire);
+ load_reserved(result, aligned_addr, operand_size::int32, acquire);
andr(scratch0, result, mask);
bne(scratch0, expected, fail);
andr(scratch0, result, scratch1); // scratch1 is ~mask
orr(scratch0, scratch0, new_val);
- sc_w(scratch0, scratch0, aligned_addr, release);
+ store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release);
bnez(scratch0, fail);
}
@@ -3997,10 +3999,10 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
void MacroAssembler::cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool) {
- assert(size != int8 && size != int16, "unsupported operand size");
+ assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size");
assert_different_registers(addr, t0);
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);
@@ -4058,10 +4060,10 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
void MacroAssembler::weak_cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result) {
-
+ assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size");
assert_different_registers(addr, t0);
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);
@@ -4134,7 +4136,7 @@ ATOMIC_XCHGU(xchgalwu, xchgalw)
#undef ATOMIC_XCHGU
void MacroAssembler::atomic_cas(Register prev, Register newv, Register addr,
- enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
+ Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
switch (size) {
case int64:
amocas_d(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
@@ -4146,6 +4148,12 @@ void MacroAssembler::atomic_cas(Register prev, Register newv, Register addr,
amocas_w(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
zext(prev, prev, 32);
break;
+ case int16:
+ amocas_h(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
+ break;
+ case int8:
+ amocas_b(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
+ break;
default:
ShouldNotReachHere();
}
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index 6f80a02ddc6..7fa7f931044 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -666,7 +666,7 @@ class MacroAssembler: public Assembler {
// We try to follow risc-v asm menomics.
// But as we don't layout a reachable GOT,
// we often need to resort to movptr, li <48imm>.
- // https://github.com/riscv-non-isa/riscv-asm-manual/blob/master/riscv-asm.md
+ // https://github.com/riscv-non-isa/riscv-asm-manual/blob/main/src/asm-manual.adoc
// Hotspot only use the standard calling convention using x1/ra.
// The alternative calling convection using x5/t0 is not used.
@@ -1187,26 +1187,26 @@ public:
void cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, Label &succeed, Label *fail);
void cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool = false);
void weak_cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result);
void cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Register shift, Register mask, Register aligned_addr);
void cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool,
Register tmp1, Register tmp2, Register tmp3);
void weak_cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result,
Register tmp1, Register tmp2, Register tmp3);
@@ -1223,7 +1223,7 @@ public:
void atomic_xchgwu(Register prev, Register newv, Register addr);
void atomic_xchgalwu(Register prev, Register newv, Register addr);
- void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size,
+ void atomic_cas(Register prev, Register newv, Register addr, Assembler::operand_size size,
Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed);
// Emit a far call/jump. Only invalidates the tmp register which
@@ -1636,8 +1636,8 @@ private:
int bitset_to_regs(unsigned int bitset, unsigned char* regs);
Address add_memory_helper(const Address dst, Register tmp);
- void load_reserved(Register dst, Register addr, enum operand_size size, Assembler::Aqrl acquire);
- void store_conditional(Register dst, Register new_val, Register addr, enum operand_size size, Assembler::Aqrl release);
+ void load_reserved(Register dst, Register addr, Assembler::operand_size size, Assembler::Aqrl acquire);
+ void store_conditional(Register dst, Register new_val, Register addr, Assembler::operand_size size, Assembler::Aqrl release);
public:
void lightweight_lock(Register basic_lock, Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow);
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index c711d2db640..0d44acc803f 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -2304,42 +2304,6 @@ encode %{
}
%}
- enc_class riscv_enc_cmpxchgw(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
- /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchgn(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
- /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchg(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
- /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchgw_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
- /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchgn_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
- /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchg_acq(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
- /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
// compare and branch instruction encodings
enc_class riscv_enc_j(label lbl) %{
@@ -2655,6 +2619,10 @@ ins_attrib ins_alignment(4); // Required alignment attribute (must
// compute_padding() function must be
// provided for the instruction
+// Whether this node is expanded during code emission into a sequence of
+// instructions and the first instruction can perform an implicit null check.
+ins_attrib ins_is_late_expanded_null_check_candidate(false);
+
//----------OPERANDS-----------------------------------------------------------
// Operand definitions must precede instruction definitions for correct parsing
// in the ADLC because operands constitute user defined types which are used in
@@ -5250,18 +5218,20 @@ instruct prefetchalloc( memory mem ) %{
// standard CompareAndSwapX when we are using barriers
// these have higher priority than the rules selected by a predicate
-instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
format %{
"cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB_narrow"
%}
ins_encode %{
@@ -5273,18 +5243,42 @@ instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R1
ins_pipe(pipe_slow);
%}
-instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
format %{
"cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS_narrow"
%}
ins_encode %{
@@ -5296,18 +5290,44 @@ instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R1
ins_pipe(pipe_slow);
%}
+instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
match(Set res (CompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapI"
%}
- ins_encode(riscv_enc_cmpxchgw(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5316,14 +5336,18 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval
%{
match(Set res (CompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapL"
%}
- ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5334,14 +5358,18 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapP"
%}
- ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5349,35 +5377,40 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
predicate(n->as_LoadStore()->barrier_data() == 0);
+
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapN"
%}
- ins_encode(riscv_enc_cmpxchgn(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
// alternative CompareAndSwapX when we are eliding barriers
-instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq_narrow"
%}
ins_encode %{
@@ -5389,20 +5422,42 @@ instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI
ins_pipe(pipe_slow);
%}
-instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq_narrow"
%}
ins_encode %{
@@ -5414,20 +5469,46 @@ instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI
ins_pipe(pipe_slow);
%}
+instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
predicate(needs_acquiring_load_reserved(n));
match(Set res (CompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapIAcq"
%}
- ins_encode(riscv_enc_cmpxchgw_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5438,14 +5519,18 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL new
match(Set res (CompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapLAcq"
%}
- ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5456,14 +5541,18 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapPAcq"
%}
- ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5474,14 +5563,18 @@ instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN new
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapNAcq"
%}
- ins_encode(riscv_enc_cmpxchgn_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5492,17 +5585,19 @@ instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN new
// no trailing StoreLoad barrier emitted by C2. Unfortunately we
// can't check the type of memory ordering here, so we always emit a
// sc_d(w) with rl bit set.
-instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB"
+ "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB_narrow"
%}
ins_encode %{
@@ -5514,17 +5609,39 @@ instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
-instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS"
+ "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS_narrow"
%}
ins_encode %{
@@ -5536,13 +5653,31 @@ instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
+instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeI"
@@ -5560,9 +5695,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne
%{
match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeL"
@@ -5579,11 +5712,10 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne
instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
predicate(n->as_LoadStore()->barrier_data() == 0);
+
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 3);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeN"
@@ -5600,11 +5732,10 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne
instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval)
%{
predicate(n->as_LoadStore()->barrier_data() == 0);
+
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeP"
@@ -5618,19 +5749,19 @@ instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP ne
ins_pipe(pipe_slow);
%}
-instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq"
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq_narrow"
%}
ins_encode %{
@@ -5642,19 +5773,39 @@ instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
-instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq"
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq_narrow"
%}
ins_encode %{
@@ -5666,15 +5817,33 @@ instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
+instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
predicate(needs_acquiring_load_reserved(n));
match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeIAcq"
@@ -5694,9 +5863,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL
match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeLAcq"
@@ -5716,9 +5883,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeNAcq"
@@ -5738,9 +5903,7 @@ instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangePAcq"
@@ -5754,18 +5917,20 @@ instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"weak_cmpxchg $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapB"
+ "# $res == 1 when success, #@weakCompareAndSwapB_narrow"
%}
ins_encode %{
@@ -5777,18 +5942,41 @@ instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapB"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"weak_cmpxchg $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapS"
+ "# $res == 1 when success, #@weakCompareAndSwapS_narrow"
%}
ins_encode %{
@@ -5800,11 +5988,32 @@ instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
+instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapS"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
@@ -5823,7 +6032,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
%{
match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
@@ -5841,9 +6050,10 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
predicate(n->as_LoadStore()->barrier_data() == 0);
+
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
@@ -5861,9 +6071,10 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne
instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval)
%{
predicate(n->as_LoadStore()->barrier_data() == 0);
+
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
@@ -5878,20 +6089,20 @@ instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"weak_cmpxchg_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapBAcq"
+ "# $res == 1 when success, #@weakCompareAndSwapBAcq_narrow"
%}
ins_encode %{
@@ -5903,20 +6114,41 @@ instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapBAcq"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"weak_cmpxchg_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapSAcq"
+ "# $res == 1 when success, #@weakCompareAndSwapSAcq_narrow"
%}
ins_encode %{
@@ -5928,13 +6160,34 @@ instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
+instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapSAcq"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
predicate(needs_acquiring_load_reserved(n));
match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg_acq $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
@@ -5955,7 +6208,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL
match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg_acq $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
@@ -5976,7 +6229,7 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
@@ -5997,7 +6250,7 @@ instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"weak_cmpxchg_acq $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
diff --git a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
index 75469ea40e9..b8de3547c83 100644
--- a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
@@ -1646,6 +1646,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
Interpreter::_remove_activation_preserving_args_entry = __ pc();
__ empty_expression_stack();
+ __ restore_bcp(); // We could have returned from deoptimizing this frame, so restore rbcp.
// Set the popframe_processing bit in pending_popframe_condition
// indicating that we are currently handling popframe, so that
// call_VMs that may happen later do not trigger new popframe
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index eca1bb83ab6..947d78477da 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -203,15 +203,15 @@ void VM_Version::common_initialize() {
}
}
- // Misc Intrinsics could depend on RVV
+ // Misc Intrinsics that could depend on RVV.
- if (UseZba || UseRVV) {
+ if (!AvoidUnalignedAccesses && (UseZba || UseRVV)) {
if (FLAG_IS_DEFAULT(UseCRC32Intrinsics)) {
FLAG_SET_DEFAULT(UseCRC32Intrinsics, true);
}
} else {
if (!FLAG_IS_DEFAULT(UseCRC32Intrinsics)) {
- warning("CRC32 intrinsic requires Zba or RVV instructions (not available on this CPU)");
+ warning("CRC32 intrinsic are not available on this CPU.");
}
FLAG_SET_DEFAULT(UseCRC32Intrinsics, false);
}
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
index 4214d6c53dc..a0a42fb5463 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
@@ -221,13 +221,13 @@ class VM_Version : public Abstract_VM_Version {
FLAG_SET_DEFAULT(UseExtension, true); \
} \
- // https://github.com/riscv/riscv-profiles/blob/main/profiles.adoc#rva20-profiles
+ // https://github.com/riscv/riscv-profiles/blob/main/src/profiles.adoc#rva20-profiles
#define RV_USE_RVA20U64 \
RV_ENABLE_EXTENSION(UseRVC) \
static void useRVA20U64Profile();
- // https://github.com/riscv/riscv-profiles/blob/main/profiles.adoc#rva22-profiles
+ // https://github.com/riscv/riscv-profiles/blob/main/src/profiles.adoc#rva22-profiles
#define RV_USE_RVA22U64 \
RV_ENABLE_EXTENSION(UseRVC) \
RV_ENABLE_EXTENSION(UseZba) \
@@ -241,7 +241,7 @@ class VM_Version : public Abstract_VM_Version {
static void useRVA22U64Profile();
- // https://github.com/riscv/riscv-profiles/blob/main/rva23-profile.adoc#rva23u64-profile
+ // https://github.com/riscv/riscv-profiles/blob/main/src/rva23-profile.adoc#rva23u64-profile
#define RV_USE_RVA23U64 \
RV_ENABLE_EXTENSION(UseRVC) \
RV_ENABLE_EXTENSION(UseRVV) \
diff --git a/src/hotspot/cpu/s390/frame_s390.hpp b/src/hotspot/cpu/s390/frame_s390.hpp
index 85e957fe992..ad754706367 100644
--- a/src/hotspot/cpu/s390/frame_s390.hpp
+++ b/src/hotspot/cpu/s390/frame_s390.hpp
@@ -410,7 +410,7 @@
// C2I adapter frames:
//
- // STACK (interpreted called from compiled, on entry to frame manager):
+ // STACK (interpreted called from compiled, on entry to template interpreter):
//
// [TOP_C2I_FRAME]
// [JIT_FRAME]
diff --git a/src/hotspot/cpu/s390/register_s390.hpp b/src/hotspot/cpu/s390/register_s390.hpp
index e7fdaa58d1a..a33145db02c 100644
--- a/src/hotspot/cpu/s390/register_s390.hpp
+++ b/src/hotspot/cpu/s390/register_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -414,7 +414,7 @@ constexpr FloatRegister Z_FARG2 = Z_F2;
constexpr FloatRegister Z_FARG3 = Z_F4;
constexpr FloatRegister Z_FARG4 = Z_F6;
-// Register declarations to be used in frame manager assembly code.
+// Register declarations to be used in template interpreter assembly code.
// Use only non-volatile registers in order to keep values across C-calls.
// Register to cache the integer value on top of the operand stack.
@@ -439,7 +439,7 @@ constexpr Register Z_bcp = Z_R13;
// Bytecode which is dispatched (short lived!).
constexpr Register Z_bytecode = Z_R14;
-// Temporary registers to be used within frame manager. We can use
+// Temporary registers to be used within template interpreter. We can use
// the nonvolatile ones because the call stub has saved them.
// Use only non-volatile registers in order to keep values across C-calls.
constexpr Register Z_tmp_1 = Z_R10;
diff --git a/src/hotspot/cpu/s390/runtime_s390.cpp b/src/hotspot/cpu/s390/runtime_s390.cpp
index 8f96ff55ccb..99a33716b8b 100644
--- a/src/hotspot/cpu/s390/runtime_s390.cpp
+++ b/src/hotspot/cpu/s390/runtime_s390.cpp
@@ -118,7 +118,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
__ z_lgr(Z_SP, saved_sp);
// [Z_RET] isn't null was possible in hotspot5 but not in sapjvm6.
- // C2I adapter extensions are now removed by a resize in the frame manager
+ // C2I adapter extensions are now removed by a resize in the template interpreter
// (unwind_initial_activation_pending_exception).
#ifdef ASSERT
__ z_ltgr(handle_exception, handle_exception);
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index bd5bbf4c7e5..cb1f12504fd 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -2139,7 +2139,7 @@ static address gen_c2i_adapter(MacroAssembler *masm,
Register value = Z_R12;
// Remember the senderSP so we can pop the interpreter arguments off of the stack.
- // In addition, frame manager expects initial_caller_sp in Z_R10.
+ // In addition, template interpreter expects initial_caller_sp in Z_R10.
__ z_lgr(sender_SP, Z_SP);
// This should always fit in 14 bit immediate.
diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
index 1e9136cdca9..d3f6540a3ea 100644
--- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
@@ -115,7 +115,7 @@ class StubGenerator: public StubCodeGenerator {
// [SP+176] - thread : Thread*
//
address generate_call_stub(address& return_address) {
- // Set up a new C frame, copy Java arguments, call frame manager
+ // Set up a new C frame, copy Java arguments, call template interpreter
// or native_entry, and process result.
StubGenStubId stub_id = StubGenStubId::call_stub_id;
@@ -272,10 +272,10 @@ class StubGenerator: public StubCodeGenerator {
BLOCK_COMMENT("call {");
{
- // Call frame manager or native entry.
+ // Call template interpreter or native entry.
//
- // Register state on entry to frame manager / native entry:
+ // Register state on entry to template interpreter / native entry:
//
// Z_ARG1 = r_top_of_arguments_addr - intptr_t *sender tos (prepushed)
// Lesp = (SP) + copied_arguments_offset - 8
@@ -290,7 +290,7 @@ class StubGenerator: public StubCodeGenerator {
__ z_lgr(Z_esp, r_top_of_arguments_addr);
//
- // Stack on entry to frame manager / native entry:
+ // Stack on entry to template interpreter / native entry:
//
// F0 [TOP_IJAVA_FRAME_ABI]
// [outgoing Java arguments]
@@ -300,7 +300,7 @@ class StubGenerator: public StubCodeGenerator {
//
// Do a light-weight C-call here, r_new_arg_entry holds the address
- // of the interpreter entry point (frame manager or native entry)
+ // of the interpreter entry point (template interpreter or native entry)
// and save runtime-value of return_pc in return_address
// (call by reference argument).
return_address = __ call_stub(r_new_arg_entry);
@@ -309,11 +309,11 @@ class StubGenerator: public StubCodeGenerator {
{
BLOCK_COMMENT("restore registers {");
- // Returned from frame manager or native entry.
+ // Returned from template interpreter or native entry.
// Now pop frame, process result, and return to caller.
//
- // Stack on exit from frame manager / native entry:
+ // Stack on exit from template interpreter / native entry:
//
// F0 [ABI]
// ...
@@ -330,7 +330,7 @@ class StubGenerator: public StubCodeGenerator {
__ pop_frame();
// Reload some volatile registers which we've spilled before the call
- // to frame manager / native entry.
+ // to template interpreter / native entry.
// Access all locals via frame pointer, because we know nothing about
// the topmost frame's size.
__ z_lg(r_arg_result_addr, result_address_offset, r_entryframe_fp);
diff --git a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
index e387d60b594..e03d891496a 100644
--- a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
@@ -1217,7 +1217,7 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
// Various method entries
-// Math function, frame manager must set up an interpreter state, etc.
+// Math function, template interpreter must set up an interpreter state, etc.
address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
// Decide what to do: Use same platform specific instructions and runtime calls as compilers.
diff --git a/src/hotspot/cpu/x86/gc/z/zAddress_x86.cpp b/src/hotspot/cpu/x86/gc/z/zAddress_x86.cpp
index 6b5b64d3036..db35a4efe08 100644
--- a/src/hotspot/cpu/x86/gc/z/zAddress_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zAddress_x86.cpp
@@ -30,11 +30,15 @@
size_t ZPointerLoadShift;
size_t ZPlatformAddressOffsetBits() {
+#ifdef ADDRESS_SANITIZER
+ return 44;
+#else
const size_t min_address_offset_bits = 42; // 4TB
const size_t max_address_offset_bits = 44; // 16TB
const size_t address_offset = ZGlobalsPointers::min_address_offset_request();
const size_t address_offset_bits = log2i_exact(address_offset);
return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
+#endif
}
size_t ZPlatformAddressHeapBaseShift() {
diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
index 045aa5d5381..e66b4b0da7a 100644
--- a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
+++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
@@ -118,6 +118,10 @@ instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)
predicate(UseZGC && n->as_Load()->barrier_data() != 0);
match(Set dst (LoadP mem));
effect(TEMP dst, KILL cr);
+ // The main load is a candidate to implement implicit null checks. The
+ // barrier's slow path includes an identical reload, which does not need to be
+ // registered in the exception table because it is dominated by the main one.
+ ins_is_late_expanded_null_check_candidate(true);
ins_cost(125);
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index 6d638ab67ef..92233ee0d07 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -1355,25 +1355,15 @@ void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
}
-void InterpreterMacroAssembler::profile_taken_branch(Register mdp,
- Register bumped_count) {
+void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
- // Otherwise, assign to mdp
test_method_data_pointer(mdp, profile_continue);
// We are taking a branch. Increment the taken count.
- // We inline increment_mdp_data_at to return bumped_count in a register
- //increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
- Address data(mdp, in_bytes(JumpData::taken_offset()));
- movptr(bumped_count, data);
- assert(DataLayout::counter_increment == 1,
- "flow-free idiom only works with 1");
- addptr(bumped_count, DataLayout::counter_increment);
- sbbptr(bumped_count, 0);
- movptr(data, bumped_count); // Store back out
+ increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
@@ -1389,7 +1379,7 @@ void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- // We are taking a branch. Increment the not taken count.
+ // We are not taking a branch. Increment the not taken count.
increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
// The method data pointer needs to be updated to correspond to
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp
index 47d54b54d7f..a36a697eebf 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.hpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp
@@ -236,7 +236,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void update_mdp_by_constant(Register mdp_in, int constant);
void update_mdp_for_ret(Register return_bci);
- void profile_taken_branch(Register mdp, Register bumped_count);
+ void profile_taken_branch(Register mdp);
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp
index d4691c9874c..bd061d45fbd 100644
--- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp
@@ -1441,6 +1441,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
Interpreter::_remove_activation_preserving_args_entry = __ pc();
__ empty_expression_stack();
+ __ restore_bcp(); // We could have returned from deoptimizing this frame, so restore rbcp.
// Set the popframe_processing bit in pending_popframe_condition
// indicating that we are currently handling popframe, so that
// call_VMs that may happen later do not trigger new popframe
diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp
index 9ea4aeeccfa..46af93e9760 100644
--- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp
@@ -465,13 +465,19 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
__ call_VM_leaf0(CAST_FROM_FN_PTR(address, SharedRuntime::dtan));
}
} else if (kind == Interpreter::java_lang_math_tanh) {
- assert(StubRoutines::dtanh() != nullptr, "not initialized");
+ if (StubRoutines::dtanh() != nullptr) {
__ movdbl(xmm0, Address(rsp, wordSize));
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::dtanh())));
+ } else {
+ return nullptr; // Fallback to default implementation
+ }
} else if (kind == Interpreter::java_lang_math_cbrt) {
- assert(StubRoutines::dcbrt() != nullptr, "not initialized");
- __ movdbl(xmm0, Address(rsp, wordSize));
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::dcbrt())));
+ if (StubRoutines::dcbrt() != nullptr) {
+ __ movdbl(xmm0, Address(rsp, wordSize));
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::dcbrt())));
+ } else {
+ return nullptr; // Fallback to default implementation
+ }
} else if (kind == Interpreter::java_lang_math_abs) {
assert(StubRoutines::x86::double_sign_mask() != nullptr, "not initialized");
__ movdbl(xmm0, Address(rsp, wordSize));
diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp
index 9f568904ae2..82ca18d8a1f 100644
--- a/src/hotspot/cpu/x86/templateTable_x86.cpp
+++ b/src/hotspot/cpu/x86/templateTable_x86.cpp
@@ -1687,8 +1687,7 @@ void TemplateTable::float_cmp(bool is_float, int unordered_result) {
void TemplateTable::branch(bool is_jsr, bool is_wide) {
__ get_method(rcx); // rcx holds method
- __ profile_taken_branch(rax, rbx); // rax holds updated MDP, rbx
- // holds bumped taken count
+ __ profile_taken_branch(rax); // rax holds updated MDP
const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
InvocationCounter::counter_offset();
@@ -1739,7 +1738,6 @@ void TemplateTable::branch(bool is_jsr, bool is_wide) {
if (UseLoopCounter) {
// increment backedge counter for backward branches
// rax: MDO
- // rbx: MDO bumped taken-count
// rcx: method
// rdx: target offset
// r13: target bcp
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index 22490ba7bb3..7b545f0f5f7 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -2055,6 +2055,10 @@ ins_attrib ins_alignment(1); // Required alignment attribute (must
// compute_padding() function must be
// provided for the instruction
+// Whether this node is expanded during code emission into a sequence of
+// instructions and the first instruction can perform an implicit null check.
+ins_attrib ins_is_late_expanded_null_check_candidate(false);
+
//----------OPERANDS-----------------------------------------------------------
// Operand definitions must precede instruction definitions for correct parsing
// in the ADLC because operands constitute user defined types which are used in
@@ -10527,7 +10531,8 @@ instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
// Xor Register with Immediate
instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
%{
- predicate(!UseAPX);
+ // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
+ predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
match(Set dst (XorI dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -10541,7 +10546,8 @@ instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
%{
- predicate(UseAPX);
+ // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
+ predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
match(Set dst (XorI src1 src2));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -10559,6 +10565,7 @@ instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
predicate(UseAPX);
match(Set dst (XorI (LoadI src1) src2));
effect(KILL cr);
+ ins_cost(150);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
@@ -11201,7 +11208,8 @@ instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
// Xor Register with Immediate
instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
%{
- predicate(!UseAPX);
+ // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
+ predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
match(Set dst (XorL dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -11215,7 +11223,8 @@ instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
%{
- predicate(UseAPX);
+ // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
+ predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
match(Set dst (XorL src1 src2));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -11234,6 +11243,7 @@ instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
match(Set dst (XorL (LoadL src1) src2));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+ ins_cost(150);
format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
ins_encode %{
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index e7b401701ac..9c6218aee16 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -1261,69 +1261,6 @@ void os::pd_print_cpu_info(outputStream* st, char* buf, size_t buflen) {
// Nothing to do beyond of what os::print_cpu_info() does.
}
-static char saved_jvm_path[MAXPATHLEN] = {0};
-
-// Find the full path to the current module, libjvm.so.
-void os::jvm_path(char *buf, jint buflen) {
- // Error checking.
- if (buflen < MAXPATHLEN) {
- assert(false, "must use a large-enough buffer");
- buf[0] = '\0';
- return;
- }
- // Lazy resolve the path to current module.
- if (saved_jvm_path[0] != 0) {
- strcpy(buf, saved_jvm_path);
- return;
- }
-
- Dl_info dlinfo;
- int ret = dladdr(CAST_FROM_FN_PTR(void *, os::jvm_path), &dlinfo);
- assert(ret != 0, "cannot locate libjvm");
- char* rp = os::realpath((char *)dlinfo.dli_fname, buf, buflen);
- assert(rp != nullptr, "error in realpath(): maybe the 'path' argument is too long?");
-
- // If executing unit tests we require JAVA_HOME to point to the real JDK.
- if (Arguments::executing_unit_tests()) {
- // Look for JAVA_HOME in the environment.
- char* java_home_var = ::getenv("JAVA_HOME");
- if (java_home_var != nullptr && java_home_var[0] != 0) {
-
- // Check the current module name "libjvm.so".
- const char* p = strrchr(buf, '/');
- if (p == nullptr) {
- return;
- }
- assert(strstr(p, "/libjvm") == p, "invalid library name");
-
- stringStream ss(buf, buflen);
- rp = os::realpath(java_home_var, buf, buflen);
- if (rp == nullptr) {
- return;
- }
-
- assert((int)strlen(buf) < buflen, "Ran out of buffer room");
- ss.print("%s/lib", buf);
-
- if (0 == access(buf, F_OK)) {
- // Use current module name "libjvm.so"
- ss.print("/%s/libjvm%s", Abstract_VM_Version::vm_variant(), JNI_LIB_SUFFIX);
- assert(strcmp(buf + strlen(buf) - strlen(JNI_LIB_SUFFIX), JNI_LIB_SUFFIX) == 0,
- "buf has been truncated");
- } else {
- // Go back to path of .so
- rp = os::realpath((char *)dlinfo.dli_fname, buf, buflen);
- if (rp == nullptr) {
- return;
- }
- }
- }
- }
-
- strncpy(saved_jvm_path, buf, sizeof(saved_jvm_path));
- saved_jvm_path[sizeof(saved_jvm_path) - 1] = '\0';
-}
-
////////////////////////////////////////////////////////////////////////////////
// Virtual Memory
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 3535d027dbc..6f7d9a6de37 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -154,7 +154,8 @@ julong os::Bsd::available_memory() {
assert(kerr == KERN_SUCCESS,
"host_statistics64 failed - check mach_host_self() and count");
if (kerr == KERN_SUCCESS) {
- available = vmstat.free_count * os::vm_page_size();
+ // free_count is just a lowerbound, other page categories can be freed too and make memory available
+ available = (vmstat.free_count + vmstat.inactive_count + vmstat.purgeable_count) * os::vm_page_size();
}
#endif
return available;
@@ -1482,83 +1483,6 @@ void os::print_memory_info(outputStream* st) {
st->cr();
}
-static char saved_jvm_path[MAXPATHLEN] = {0};
-
-// Find the full path to the current module, libjvm
-void os::jvm_path(char *buf, jint buflen) {
- // Error checking.
- if (buflen < MAXPATHLEN) {
- assert(false, "must use a large-enough buffer");
- buf[0] = '\0';
- return;
- }
- // Lazy resolve the path to current module.
- if (saved_jvm_path[0] != 0) {
- strcpy(buf, saved_jvm_path);
- return;
- }
-
- char dli_fname[MAXPATHLEN];
- dli_fname[0] = '\0';
- bool ret = dll_address_to_library_name(
- CAST_FROM_FN_PTR(address, os::jvm_path),
- dli_fname, sizeof(dli_fname), nullptr);
- assert(ret, "cannot locate libjvm");
- char *rp = nullptr;
- if (ret && dli_fname[0] != '\0') {
- rp = os::realpath(dli_fname, buf, buflen);
- }
- if (rp == nullptr) {
- return;
- }
-
- // If executing unit tests we require JAVA_HOME to point to the real JDK.
- if (Arguments::executing_unit_tests()) {
- // Look for JAVA_HOME in the environment.
- char* java_home_var = ::getenv("JAVA_HOME");
- if (java_home_var != nullptr && java_home_var[0] != 0) {
-
- // Check the current module name "libjvm"
- const char* p = strrchr(buf, '/');
- assert(strstr(p, "/libjvm") == p, "invalid library name");
-
- stringStream ss(buf, buflen);
- rp = os::realpath(java_home_var, buf, buflen);
- if (rp == nullptr) {
- return;
- }
-
- assert((int)strlen(buf) < buflen, "Ran out of buffer space");
- // Add the appropriate library and JVM variant subdirs
- ss.print("%s/lib/%s", buf, Abstract_VM_Version::vm_variant());
-
- if (0 != access(buf, F_OK)) {
- ss.reset();
- ss.print("%s/lib", buf);
- }
-
- // If the path exists within JAVA_HOME, add the JVM library name
- // to complete the path to JVM being overridden. Otherwise fallback
- // to the path to the current library.
- if (0 == access(buf, F_OK)) {
- // Use current module name "libjvm"
- ss.print("/libjvm%s", JNI_LIB_SUFFIX);
- assert(strcmp(buf + strlen(buf) - strlen(JNI_LIB_SUFFIX), JNI_LIB_SUFFIX) == 0,
- "buf has been truncated");
- } else {
- // Fall back to path of current library
- rp = os::realpath(dli_fname, buf, buflen);
- if (rp == nullptr) {
- return;
- }
- }
- }
- }
-
- strncpy(saved_jvm_path, buf, MAXPATHLEN);
- saved_jvm_path[MAXPATHLEN - 1] = '\0';
-}
-
////////////////////////////////////////////////////////////////////////////////
// Virtual Memory
diff --git a/src/hotspot/os/linux/globals_linux.hpp b/src/hotspot/os/linux/globals_linux.hpp
index 331690ec3ee..542e034f59f 100644
--- a/src/hotspot/os/linux/globals_linux.hpp
+++ b/src/hotspot/os/linux/globals_linux.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,9 +35,6 @@
range, \
constraint) \
\
- product(bool, UseOprofile, false, \
- "(Deprecated) enable support for Oprofile profiler") \
- \
product(bool, UseTransparentHugePages, false, \
"Use MADV_HUGEPAGE for large pages") \
\
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 807014e7b0c..ff2195d3839 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -2746,118 +2746,9 @@ void os::get_summary_cpu_info(char* cpuinfo, size_t length) {
#endif
}
-static char saved_jvm_path[MAXPATHLEN] = {0};
-
-// Find the full path to the current module, libjvm.so
-void os::jvm_path(char *buf, jint buflen) {
- // Error checking.
- if (buflen < MAXPATHLEN) {
- assert(false, "must use a large-enough buffer");
- buf[0] = '\0';
- return;
- }
- // Lazy resolve the path to current module.
- if (saved_jvm_path[0] != 0) {
- strcpy(buf, saved_jvm_path);
- return;
- }
-
- char dli_fname[MAXPATHLEN];
- dli_fname[0] = '\0';
- bool ret = dll_address_to_library_name(
- CAST_FROM_FN_PTR(address, os::jvm_path),
- dli_fname, sizeof(dli_fname), nullptr);
- assert(ret, "cannot locate libjvm");
- char *rp = nullptr;
- if (ret && dli_fname[0] != '\0') {
- rp = os::realpath(dli_fname, buf, buflen);
- }
- if (rp == nullptr) {
- return;
- }
-
- // If executing unit tests we require JAVA_HOME to point to the real JDK.
- if (Arguments::executing_unit_tests()) {
- // Look for JAVA_HOME in the environment.
- char* java_home_var = ::getenv("JAVA_HOME");
- if (java_home_var != nullptr && java_home_var[0] != 0) {
-
- // Check the current module name "libjvm.so".
- const char* p = strrchr(buf, '/');
- if (p == nullptr) {
- return;
- }
- assert(strstr(p, "/libjvm") == p, "invalid library name");
-
- stringStream ss(buf, buflen);
- rp = os::realpath(java_home_var, buf, buflen);
- if (rp == nullptr) {
- return;
- }
-
- assert((int)strlen(buf) < buflen, "Ran out of buffer room");
- ss.print("%s/lib", buf);
-
- if (0 == access(buf, F_OK)) {
- // Use current module name "libjvm.so"
- ss.print("/%s/libjvm%s", Abstract_VM_Version::vm_variant(), JNI_LIB_SUFFIX);
- assert(strcmp(buf + strlen(buf) - strlen(JNI_LIB_SUFFIX), JNI_LIB_SUFFIX) == 0,
- "buf has been truncated");
- } else {
- // Go back to path of .so
- rp = os::realpath(dli_fname, buf, buflen);
- if (rp == nullptr) {
- return;
- }
- }
- }
- }
-
- strncpy(saved_jvm_path, buf, MAXPATHLEN);
- saved_jvm_path[MAXPATHLEN - 1] = '\0';
-}
-
////////////////////////////////////////////////////////////////////////////////
// Virtual Memory
-// Rationale behind this function:
-// current (Mon Apr 25 20:12:18 MSD 2005) oprofile drops samples without executable
-// mapping for address (see lookup_dcookie() in the kernel module), thus we cannot get
-// samples for JITted code. Here we create private executable mapping over the code cache
-// and then we can use standard (well, almost, as mapping can change) way to provide
-// info for the reporting script by storing timestamp and location of symbol
-void linux_wrap_code(char* base, size_t size) {
- static volatile jint cnt = 0;
-
- static_assert(sizeof(off_t) == 8, "Expected Large File Support in this file");
-
- if (!UseOprofile) {
- return;
- }
-
- char buf[PATH_MAX+1];
- int num = Atomic::add(&cnt, 1);
-
- snprintf(buf, sizeof(buf), "%s/hs-vm-%d-%d",
- os::get_temp_directory(), os::current_process_id(), num);
- unlink(buf);
-
- int fd = ::open(buf, O_CREAT | O_RDWR, S_IRWXU);
-
- if (fd != -1) {
- off_t rv = ::lseek(fd, size-2, SEEK_SET);
- if (rv != (off_t)-1) {
- if (::write(fd, "", 1) == 1) {
- mmap(base, size,
- PROT_READ|PROT_WRITE|PROT_EXEC,
- MAP_PRIVATE|MAP_FIXED|MAP_NORESERVE, fd, 0);
- }
- }
- ::close(fd);
- unlink(buf);
- }
-}
-
static bool recoverable_mmap_error(int err) {
// See if the error is one we can let the caller handle. This
// list of errno values comes from JBS-6843484. I can't find a
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 448ebce620a..303e44eadcb 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -59,6 +59,7 @@
#ifdef AIX
#include "loadlib_aix.hpp"
#include "os_aix.hpp"
+#include "porting_aix.hpp"
#endif
#ifdef LINUX
#include "os_linux.hpp"
@@ -1060,6 +1061,95 @@ bool os::same_files(const char* file1, const char* file2) {
return is_same;
}
+static char saved_jvm_path[MAXPATHLEN] = {0};
+
+// Find the full path to the current module, libjvm.so
+void os::jvm_path(char *buf, jint buflen) {
+ // Error checking.
+ if (buflen < MAXPATHLEN) {
+ assert(false, "must use a large-enough buffer");
+ buf[0] = '\0';
+ return;
+ }
+ // Lazy resolve the path to current module.
+ if (saved_jvm_path[0] != 0) {
+ strcpy(buf, saved_jvm_path);
+ return;
+ }
+
+ const char* fname;
+#ifdef AIX
+ Dl_info dlinfo;
+ int ret = dladdr(CAST_FROM_FN_PTR(void *, os::jvm_path), &dlinfo);
+ assert(ret != 0, "cannot locate libjvm");
+ if (ret == 0) {
+ return;
+ }
+ fname = dlinfo.dli_fname;
+#else
+ char dli_fname[MAXPATHLEN];
+ dli_fname[0] = '\0';
+ bool ret = dll_address_to_library_name(
+ CAST_FROM_FN_PTR(address, os::jvm_path),
+ dli_fname, sizeof(dli_fname), nullptr);
+ assert(ret, "cannot locate libjvm");
+ if (!ret) {
+ return;
+ }
+ fname = dli_fname;
+#endif // AIX
+ char* rp = nullptr;
+ if (fname[0] != '\0') {
+ rp = os::realpath(fname, buf, buflen);
+ }
+ if (rp == nullptr) {
+ return;
+ }
+
+ // If executing unit tests we require JAVA_HOME to point to the real JDK.
+ if (Arguments::executing_unit_tests()) {
+ // Look for JAVA_HOME in the environment.
+ char* java_home_var = ::getenv("JAVA_HOME");
+ if (java_home_var != nullptr && java_home_var[0] != 0) {
+
+ // Check the current module name "libjvm.so".
+ const char* p = strrchr(buf, '/');
+ if (p == nullptr) {
+ return;
+ }
+ assert(strstr(p, "/libjvm") == p, "invalid library name");
+
+ stringStream ss(buf, buflen);
+ rp = os::realpath(java_home_var, buf, buflen);
+ if (rp == nullptr) {
+ return;
+ }
+
+ assert((int)strlen(buf) < buflen, "Ran out of buffer room");
+ ss.print("%s/lib", buf);
+
+ // If the path exists within JAVA_HOME, add the VM variant directory and JVM
+ // library name to complete the path to JVM being overridden. Otherwise fallback
+ // to the path to the current library.
+ if (0 == access(buf, F_OK)) {
+ // Use current module name "libjvm.so"
+ ss.print("/%s/libjvm%s", Abstract_VM_Version::vm_variant(), JNI_LIB_SUFFIX);
+ assert(strcmp(buf + strlen(buf) - strlen(JNI_LIB_SUFFIX), JNI_LIB_SUFFIX) == 0,
+ "buf has been truncated");
+ } else {
+ // Go back to path of .so
+ rp = os::realpath(fname, buf, buflen);
+ if (rp == nullptr) {
+ return;
+ }
+ }
+ }
+ }
+
+ strncpy(saved_jvm_path, buf, MAXPATHLEN);
+ saved_jvm_path[MAXPATHLEN - 1] = '\0';
+}
+
// Called when creating the thread. The minimum stack sizes have already been calculated
size_t os::Posix::get_initial_stack_size(ThreadType thr_type, size_t req_stack_size) {
size_t stack_size;
diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp
index e900d5695ae..0157d354f40 100644
--- a/src/hotspot/os/posix/signals_posix.cpp
+++ b/src/hotspot/os/posix/signals_posix.cpp
@@ -1505,6 +1505,14 @@ bool PosixSignals::is_sig_ignored(int sig) {
}
}
+void* PosixSignals::get_signal_handler_for_signal(int sig) {
+ struct sigaction oact;
+ if (sigaction(sig, (struct sigaction*)nullptr, &oact) == -1) {
+ return nullptr;
+ }
+ return get_signal_handler(&oact);
+}
+
static void signal_sets_init() {
sigemptyset(&preinstalled_sigs);
diff --git a/src/hotspot/os/posix/signals_posix.hpp b/src/hotspot/os/posix/signals_posix.hpp
index 9deade4db18..c1cb70df153 100644
--- a/src/hotspot/os/posix/signals_posix.hpp
+++ b/src/hotspot/os/posix/signals_posix.hpp
@@ -52,6 +52,8 @@ public:
static bool is_sig_ignored(int sig);
+ static void* get_signal_handler_for_signal(int sig);
+
static void hotspot_sigmask(Thread* thread);
static void print_signal_handler(outputStream* st, int sig, char* buf, size_t buflen);
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index ae352641516..ad32ee150e8 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -81,14 +81,12 @@
#endif
#define SPELL_REG_SP "sp"
-#define SPELL_REG_FP "fp"
#ifdef __APPLE__
// see darwin-xnu/osfmk/mach/arm/_structs.h
// 10.5 UNIX03 member name prefixes
#define DU3_PREFIX(s, m) __ ## s.__ ## m
-#endif
#define context_x uc_mcontext->DU3_PREFIX(ss,x)
#define context_fp uc_mcontext->DU3_PREFIX(ss,fp)
@@ -97,6 +95,31 @@
#define context_pc uc_mcontext->DU3_PREFIX(ss,pc)
#define context_cpsr uc_mcontext->DU3_PREFIX(ss,cpsr)
#define context_esr uc_mcontext->DU3_PREFIX(es,esr)
+#endif
+
+#ifdef __FreeBSD__
+# define context_x uc_mcontext.mc_gpregs.gp_x
+# define context_fp context_x[REG_FP]
+# define context_lr uc_mcontext.mc_gpregs.gp_lr
+# define context_sp uc_mcontext.mc_gpregs.gp_sp
+# define context_pc uc_mcontext.mc_gpregs.gp_elr
+#endif
+
+#ifdef __NetBSD__
+# define context_x uc_mcontext.__gregs
+# define context_fp uc_mcontext.__gregs[_REG_FP]
+# define context_lr uc_mcontext.__gregs[_REG_LR]
+# define context_sp uc_mcontext.__gregs[_REG_SP]
+# define context_pc uc_mcontext.__gregs[_REG_ELR]
+#endif
+
+#ifdef __OpenBSD__
+# define context_x sc_x
+# define context_fp sc_x[REG_FP]
+# define context_lr sc_lr
+# define context_sp sc_sp
+# define context_pc sc_elr
+#endif
#define REG_BCP context_x[22]
@@ -497,9 +520,11 @@ int os::extra_bang_size_in_bytes() {
return 0;
}
+#ifdef __APPLE__
void os::current_thread_enable_wx(WXMode mode) {
pthread_jit_write_protect_np(mode == WXExec);
}
+#endif
static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
*(jlong *) dst = *(const jlong *) src;
diff --git a/src/hotspot/share/adlc/main.cpp b/src/hotspot/share/adlc/main.cpp
index 16fd4ddcf93..4e8a96617e8 100644
--- a/src/hotspot/share/adlc/main.cpp
+++ b/src/hotspot/share/adlc/main.cpp
@@ -481,7 +481,3 @@ int get_legal_text(FileBuff &fbuf, char **legal_text)
*legal_text = legal_start;
return (int) (legal_end - legal_start);
}
-
-void *operator new( size_t size, int, const char *, int ) throw() {
- return ::operator new( size );
-}
diff --git a/src/hotspot/share/adlc/output_h.cpp b/src/hotspot/share/adlc/output_h.cpp
index b62bc43791f..cbcc00efa3b 100644
--- a/src/hotspot/share/adlc/output_h.cpp
+++ b/src/hotspot/share/adlc/output_h.cpp
@@ -1626,6 +1626,8 @@ void ArchDesc::declareClasses(FILE *fp) {
while (attr != nullptr) {
if (strcmp (attr->_ident, "ins_is_TrapBasedCheckNode") == 0) {
fprintf(fp, " virtual bool is_TrapBasedCheckNode() const { return %s; }\n", attr->_val);
+ } else if (strcmp (attr->_ident, "ins_is_late_expanded_null_check_candidate") == 0) {
+ fprintf(fp, " virtual bool is_late_expanded_null_check_candidate() const { return %s; }\n", attr->_val);
} else if (strcmp (attr->_ident, "ins_cost") != 0 &&
strncmp(attr->_ident, "ins_field_", 10) != 0 &&
// Must match function in node.hpp: return type bool, no prefix "ins_".
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index ca25cf56be0..81bfc932147 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -29,6 +29,7 @@
#include "compiler/disassembler.hpp"
#include "logging/log.hpp"
#include "oops/klass.inline.hpp"
+#include "oops/methodCounters.hpp"
#include "oops/methodData.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/icache.hpp"
@@ -537,6 +538,9 @@ void CodeBuffer::finalize_oop_references(const methodHandle& mh) {
if (m->is_methodData()) {
m = ((MethodData*)m)->method();
}
+ if (m->is_methodCounters()) {
+ m = ((MethodCounters*)m)->method();
+ }
if (m->is_method()) {
m = ((Method*)m)->method_holder();
}
@@ -561,6 +565,9 @@ void CodeBuffer::finalize_oop_references(const methodHandle& mh) {
if (m->is_methodData()) {
m = ((MethodData*)m)->method();
}
+ if (m->is_methodCounters()) {
+ m = ((MethodCounters*)m)->method();
+ }
if (m->is_method()) {
m = ((Method*)m)->method_holder();
}
@@ -1099,7 +1106,8 @@ CHeapString::~CHeapString() {
// offset is a byte offset into an instruction stream (CodeBuffer, CodeBlob or
// other memory buffer) and remark is a string (comment).
//
-AsmRemarks::AsmRemarks() : _remarks(new AsmRemarkCollection()) {
+AsmRemarks::AsmRemarks() {
+ init();
assert(_remarks != nullptr, "Allocation failure!");
}
@@ -1107,6 +1115,10 @@ AsmRemarks::~AsmRemarks() {
assert(_remarks == nullptr, "Must 'clear()' before deleting!");
}
+void AsmRemarks::init() {
+ _remarks = new AsmRemarkCollection();
+}
+
const char* AsmRemarks::insert(uint offset, const char* remstr) {
precond(remstr != nullptr);
return _remarks->insert(offset, remstr);
@@ -1151,7 +1163,8 @@ uint AsmRemarks::print(uint offset, outputStream* strm) const {
// Acting as interface to reference counted collection of (debug) strings used
// in the code generated, and thus requiring a fixed address.
//
-DbgStrings::DbgStrings() : _strings(new DbgStringCollection()) {
+DbgStrings::DbgStrings() {
+ init();
assert(_strings != nullptr, "Allocation failure!");
}
@@ -1159,6 +1172,10 @@ DbgStrings::~DbgStrings() {
assert(_strings == nullptr, "Must 'clear()' before deleting!");
}
+void DbgStrings::init() {
+ _strings = new DbgStringCollection();
+}
+
const char* DbgStrings::insert(const char* dbgstr) {
const char* str = _strings->lookup(dbgstr);
return str != nullptr ? str : _strings->insert(dbgstr);
diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp
index e6dac484649..96e9a77a923 100644
--- a/src/hotspot/share/asm/codeBuffer.hpp
+++ b/src/hotspot/share/asm/codeBuffer.hpp
@@ -426,6 +426,8 @@ class AsmRemarks {
AsmRemarks();
~AsmRemarks();
+ void init();
+
const char* insert(uint offset, const char* remstr);
bool is_empty() const;
@@ -452,6 +454,8 @@ class DbgStrings {
DbgStrings();
~DbgStrings();
+ void init();
+
const char* insert(const char* dbgstr);
bool is_empty() const;
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 7c41c09d378..e2760689daa 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -818,7 +818,7 @@ JRT_ENTRY(void, Runtime1::deoptimize(JavaThread* current, jint trap_request))
Deoptimization::DeoptReason reason = Deoptimization::trap_request_reason(trap_request);
if (action == Deoptimization::Action_make_not_entrant) {
- if (nm->make_not_entrant("C1 deoptimize")) {
+ if (nm->make_not_entrant(nmethod::ChangeReason::C1_deoptimize)) {
if (reason == Deoptimization::Reason_tenured) {
MethodData* trap_mdo = Deoptimization::get_method_data(current, method, true /*create_if_missing*/);
if (trap_mdo != nullptr) {
@@ -1110,7 +1110,7 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, C1StubId stub_id ))
// safepoint, but if it's still alive then make it not_entrant.
nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
if (nm != nullptr) {
- nm->make_not_entrant("C1 code patch");
+ nm->make_not_entrant(nmethod::ChangeReason::C1_codepatch);
}
Deoptimization::deoptimize_frame(current, caller_frame.id());
@@ -1358,7 +1358,7 @@ void Runtime1::patch_code(JavaThread* current, C1StubId stub_id) {
// Make sure the nmethod is invalidated, i.e. made not entrant.
nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
if (nm != nullptr) {
- nm->make_not_entrant("C1 deoptimize for patching");
+ nm->make_not_entrant(nmethod::ChangeReason::C1_deoptimize_for_patching);
}
}
@@ -1486,7 +1486,7 @@ JRT_ENTRY(void, Runtime1::predicate_failed_trap(JavaThread* current))
nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
assert (nm != nullptr, "no more nmethod?");
- nm->make_not_entrant("C1 predicate failed trap");
+ nm->make_not_entrant(nmethod::ChangeReason::C1_predicate_failed_trap);
methodHandle m(current, nm->method());
MethodData* mdo = m->method_data();
diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp
index b3c13b63aff..b93238eca88 100644
--- a/src/hotspot/share/cds/cdsConfig.cpp
+++ b/src/hotspot/share/cds/cdsConfig.cpp
@@ -110,12 +110,24 @@ const char* CDSConfig::default_archive_path() {
// before CDSConfig::ergo_initialize() is called.
assert(_cds_ergo_initialize_started, "sanity");
if (_default_archive_path == nullptr) {
- char jvm_path[JVM_MAXPATHLEN];
- os::jvm_path(jvm_path, sizeof(jvm_path));
- char *end = strrchr(jvm_path, *os::file_separator());
- if (end != nullptr) *end = '\0';
stringStream tmp;
- tmp.print("%s%sclasses", jvm_path, os::file_separator());
+ if (is_vm_statically_linked()) {
+ // It's easier to form the path using JAVA_HOME as os::jvm_path
+ // gives the path to the launcher executable on static JDK.
+ const char* subdir = WINDOWS_ONLY("bin") NOT_WINDOWS("lib");
+ tmp.print("%s%s%s%s%s%sclasses",
+ Arguments::get_java_home(), os::file_separator(),
+ subdir, os::file_separator(),
+ Abstract_VM_Version::vm_variant(), os::file_separator());
+ } else {
+ // Assume .jsa is in the same directory where libjvm resides on
+ // non-static JDK.
+ char jvm_path[JVM_MAXPATHLEN];
+ os::jvm_path(jvm_path, sizeof(jvm_path));
+ char *end = strrchr(jvm_path, *os::file_separator());
+ if (end != nullptr) *end = '\0';
+ tmp.print("%s%sclasses", jvm_path, os::file_separator());
+ }
#ifdef _LP64
if (!UseCompressedOops) {
tmp.print_raw("_nocoops");
diff --git a/src/hotspot/share/ci/ciReplay.cpp b/src/hotspot/share/ci/ciReplay.cpp
index 5ea4336f35e..f9829e88c4a 100644
--- a/src/hotspot/share/ci/ciReplay.cpp
+++ b/src/hotspot/share/ci/ciReplay.cpp
@@ -802,7 +802,7 @@ class CompileReplay : public StackObj {
// Make sure the existence of a prior compile doesn't stop this one
nmethod* nm = (entry_bci != InvocationEntryBci) ? method->lookup_osr_nmethod_for(entry_bci, comp_level, true) : method->code();
if (nm != nullptr) {
- nm->make_not_entrant("CI replay");
+ nm->make_not_entrant(nmethod::ChangeReason::CI_replay);
}
replay_state = this;
CompileBroker::compile_method(methodHandle(THREAD, method), entry_bci, comp_level,
diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp
index 0f021c0ef4d..edec019bd70 100644
--- a/src/hotspot/share/classfile/classFileParser.cpp
+++ b/src/hotspot/share/classfile/classFileParser.cpp
@@ -154,6 +154,8 @@
#define JAVA_25_VERSION 69
+#define JAVA_26_VERSION 70
+
void ClassFileParser::set_class_bad_constant_seen(short bad_constant) {
assert((bad_constant == JVM_CONSTANT_Module ||
bad_constant == JVM_CONSTANT_Package) && _major_version >= JAVA_9_VERSION,
@@ -3738,6 +3740,7 @@ void ClassFileParser::apply_parsed_class_metadata(
_cp->set_pool_holder(this_klass);
this_klass->set_constants(_cp);
this_klass->set_fieldinfo_stream(_fieldinfo_stream);
+ this_klass->set_fieldinfo_search_table(_fieldinfo_search_table);
this_klass->set_fields_status(_fields_status);
this_klass->set_methods(_methods);
this_klass->set_inner_classes(_inner_classes);
@@ -3747,6 +3750,8 @@ void ClassFileParser::apply_parsed_class_metadata(
this_klass->set_permitted_subclasses(_permitted_subclasses);
this_klass->set_record_components(_record_components);
+ DEBUG_ONLY(FieldInfoStream::validate_search_table(_cp, _fieldinfo_stream, _fieldinfo_search_table));
+
// Delay the setting of _local_interfaces and _transitive_interfaces until after
// initialize_supers() in fill_instance_klass(). It is because the _local_interfaces could
// be shared with _transitive_interfaces and _transitive_interfaces may be shared with
@@ -5054,6 +5059,7 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik,
// note that is not safe to use the fields in the parser from this point on
assert(nullptr == _cp, "invariant");
assert(nullptr == _fieldinfo_stream, "invariant");
+ assert(nullptr == _fieldinfo_search_table, "invariant");
assert(nullptr == _fields_status, "invariant");
assert(nullptr == _methods, "invariant");
assert(nullptr == _inner_classes, "invariant");
@@ -5274,6 +5280,7 @@ ClassFileParser::ClassFileParser(ClassFileStream* stream,
_super_klass(),
_cp(nullptr),
_fieldinfo_stream(nullptr),
+ _fieldinfo_search_table(nullptr),
_fields_status(nullptr),
_methods(nullptr),
_inner_classes(nullptr),
@@ -5350,6 +5357,7 @@ void ClassFileParser::clear_class_metadata() {
// deallocated if classfile parsing returns an error.
_cp = nullptr;
_fieldinfo_stream = nullptr;
+ _fieldinfo_search_table = nullptr;
_fields_status = nullptr;
_methods = nullptr;
_inner_classes = nullptr;
@@ -5372,6 +5380,7 @@ ClassFileParser::~ClassFileParser() {
if (_fieldinfo_stream != nullptr) {
MetadataFactory::free_array(_loader_data, _fieldinfo_stream);
}
+ MetadataFactory::free_array(_loader_data, _fieldinfo_search_table);
if (_fields_status != nullptr) {
MetadataFactory::free_array(_loader_data, _fields_status);
@@ -5772,6 +5781,7 @@ void ClassFileParser::post_process_parsed_stream(const ClassFileStream* const st
_fieldinfo_stream =
FieldInfoStream::create_FieldInfoStream(_temp_field_info, _java_fields_count,
injected_fields_count, loader_data(), CHECK);
+ _fieldinfo_search_table = FieldInfoStream::create_search_table(_cp, _fieldinfo_stream, _loader_data, CHECK);
_fields_status =
MetadataFactory::new_array(_loader_data, _temp_field_info->length(),
FieldStatus(0), CHECK);
diff --git a/src/hotspot/share/classfile/classFileParser.hpp b/src/hotspot/share/classfile/classFileParser.hpp
index 8f9f4ebea4d..707fbf6985f 100644
--- a/src/hotspot/share/classfile/classFileParser.hpp
+++ b/src/hotspot/share/classfile/classFileParser.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -123,6 +123,7 @@ class ClassFileParser {
const InstanceKlass* _super_klass;
ConstantPool* _cp;
Array* _fieldinfo_stream;
+ Array* _fieldinfo_search_table;
Array* _fields_status;
Array* _methods;
Array* _inner_classes;
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
index b6007923907..03afe89f4f8 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
@@ -301,7 +301,7 @@ void FieldLayout::reconstruct_layout(const InstanceKlass* ik, bool& has_instance
BasicType last_type;
int last_offset = -1;
while (ik != nullptr) {
- for (AllFieldStream fs(ik->fieldinfo_stream(), ik->constants()); !fs.done(); fs.next()) {
+ for (AllFieldStream fs(ik); !fs.done(); fs.next()) {
BasicType type = Signature::basic_type(fs.signature());
// distinction between static and non-static fields is missing
if (fs.access_flags().is_static()) continue;
@@ -461,7 +461,7 @@ void FieldLayout::print(outputStream* output, bool is_static, const InstanceKlas
bool found = false;
const InstanceKlass* ik = super;
while (!found && ik != nullptr) {
- for (AllFieldStream fs(ik->fieldinfo_stream(), ik->constants()); !fs.done(); fs.next()) {
+ for (AllFieldStream fs(ik); !fs.done(); fs.next()) {
if (fs.offset() == b->offset()) {
output->print_cr(" @%d \"%s\" %s %d/%d %s",
b->offset(),
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index 3b34edf416e..0e8467289ad 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -967,6 +967,13 @@ void java_lang_Class::fixup_mirror(Klass* k, TRAPS) {
Array* new_fis = FieldInfoStream::create_FieldInfoStream(fields, java_fields, injected_fields, k->class_loader_data(), CHECK);
ik->set_fieldinfo_stream(new_fis);
MetadataFactory::free_array(k->class_loader_data(), old_stream);
+
+ Array* old_table = ik->fieldinfo_search_table();
+ Array* search_table = FieldInfoStream::create_search_table(ik->constants(), new_fis, k->class_loader_data(), CHECK);
+ ik->set_fieldinfo_search_table(search_table);
+ MetadataFactory::free_array(k->class_loader_data(), old_table);
+
+ DEBUG_ONLY(FieldInfoStream::validate_search_table(ik->constants(), new_fis, search_table));
}
}
@@ -1872,7 +1879,7 @@ ByteSize java_lang_Thread::thread_id_offset() {
}
oop java_lang_Thread::park_blocker(oop java_thread) {
- return java_thread->obj_field(_park_blocker_offset);
+ return java_thread->obj_field_access(_park_blocker_offset);
}
oop java_lang_Thread::async_get_stack_trace(oop java_thread, TRAPS) {
diff --git a/src/hotspot/share/classfile/verificationType.cpp b/src/hotspot/share/classfile/verificationType.cpp
index e6115dfce9b..aedb620aabe 100644
--- a/src/hotspot/share/classfile/verificationType.cpp
+++ b/src/hotspot/share/classfile/verificationType.cpp
@@ -48,41 +48,50 @@ VerificationType VerificationType::from_tag(u1 tag) {
}
}
-bool VerificationType::resolve_and_check_assignability(InstanceKlass* klass, Symbol* name,
- Symbol* from_name, bool from_field_is_protected, bool from_is_array, bool from_is_object, TRAPS) {
+// Potentially resolve the target class and from class, and check whether the from class is assignable
+// to the target class. The current_klass is the class being verified - it could also be the target in some
+// cases, and otherwise is needed to obtain the correct classloader for resolving the other classes.
+bool VerificationType::resolve_and_check_assignability(InstanceKlass* current_klass, Symbol* target_name, Symbol* from_name,
+ bool from_field_is_protected, bool from_is_array,
+ bool from_is_object, bool* target_is_interface, TRAPS) {
HandleMark hm(THREAD);
- Klass* this_class;
- if (klass->is_hidden() && klass->name() == name) {
- this_class = klass;
+ Klass* target_klass;
+ if (current_klass->is_hidden() && current_klass->name() == target_name) {
+ target_klass = current_klass;
} else {
- this_class = SystemDictionary::resolve_or_fail(
- name, Handle(THREAD, klass->class_loader()), true, CHECK_false);
+ target_klass = SystemDictionary::resolve_or_fail(
+ target_name, Handle(THREAD, current_klass->class_loader()), true, CHECK_false);
if (log_is_enabled(Debug, class, resolve)) {
- Verifier::trace_class_resolution(this_class, klass);
+ Verifier::trace_class_resolution(target_klass, current_klass);
}
}
- if (this_class->is_interface() && (!from_field_is_protected ||
+ bool is_intf = target_klass->is_interface();
+ if (target_is_interface != nullptr) {
+ *target_is_interface = is_intf;
+ }
+
+ if (is_intf && (!from_field_is_protected ||
from_name != vmSymbols::java_lang_Object())) {
// If we are not trying to access a protected field or method in
// java.lang.Object then, for arrays, we only allow assignability
// to interfaces java.lang.Cloneable and java.io.Serializable.
// Otherwise, we treat interfaces as java.lang.Object.
return !from_is_array ||
- this_class == vmClasses::Cloneable_klass() ||
- this_class == vmClasses::Serializable_klass();
+ target_klass == vmClasses::Cloneable_klass() ||
+ target_klass == vmClasses::Serializable_klass();
} else if (from_is_object) {
- Klass* from_class;
- if (klass->is_hidden() && klass->name() == from_name) {
- from_class = klass;
+ Klass* from_klass;
+ if (current_klass->is_hidden() && current_klass->name() == from_name) {
+ from_klass = current_klass;
} else {
- from_class = SystemDictionary::resolve_or_fail(
- from_name, Handle(THREAD, klass->class_loader()), true, CHECK_false);
+ from_klass = SystemDictionary::resolve_or_fail(
+ from_name, Handle(THREAD, current_klass->class_loader()), true, CHECK_false);
if (log_is_enabled(Debug, class, resolve)) {
- Verifier::trace_class_resolution(from_class, klass);
+ Verifier::trace_class_resolution(from_klass, current_klass);
}
}
- return from_class->is_subclass_of(this_class);
+ return from_klass->is_subclass_of(target_klass);
}
return false;
@@ -90,8 +99,8 @@ bool VerificationType::resolve_and_check_assignability(InstanceKlass* klass, Sym
bool VerificationType::is_reference_assignable_from(
const VerificationType& from, ClassVerifier* context,
- bool from_field_is_protected, TRAPS) const {
- InstanceKlass* klass = context->current_class();
+ bool from_field_is_protected, bool* this_is_interface, TRAPS) const {
+
if (from.is_null()) {
// null is assignable to any reference
return true;
@@ -109,7 +118,7 @@ bool VerificationType::is_reference_assignable_from(
#if INCLUDE_CDS
if (CDSConfig::is_dumping_archive()) {
bool skip_assignability_check = false;
- SystemDictionaryShared::add_verification_constraint(klass,
+ SystemDictionaryShared::add_verification_constraint(context->current_class(),
name(), from.name(), from_field_is_protected, from.is_array(),
from.is_object(), &skip_assignability_check);
if (skip_assignability_check) {
@@ -119,8 +128,9 @@ bool VerificationType::is_reference_assignable_from(
}
}
#endif
- return resolve_and_check_assignability(klass, name(), from.name(),
- from_field_is_protected, from.is_array(), from.is_object(), THREAD);
+ return resolve_and_check_assignability(context->current_class(), name(), from.name(),
+ from_field_is_protected, from.is_array(),
+ from.is_object(), this_is_interface, THREAD);
} else if (is_array() && from.is_array()) {
VerificationType comp_this = get_component(context);
VerificationType comp_from = from.get_component(context);
diff --git a/src/hotspot/share/classfile/verificationType.hpp b/src/hotspot/share/classfile/verificationType.hpp
index 4f0609f0cce..788e0029fad 100644
--- a/src/hotspot/share/classfile/verificationType.hpp
+++ b/src/hotspot/share/classfile/verificationType.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -288,7 +288,7 @@ class VerificationType {
if (is_reference() && from.is_reference()) {
return is_reference_assignable_from(from, context,
from_field_is_protected,
- THREAD);
+ nullptr, THREAD);
} else {
return false;
}
@@ -327,17 +327,24 @@ class VerificationType {
void print_on(outputStream* st) const;
- private:
+ bool is_reference_assignable_from(const VerificationType& from, ClassVerifier* context,
+ bool from_field_is_protected, bool* this_is_interface, TRAPS) const;
- bool is_reference_assignable_from(
- const VerificationType&, ClassVerifier*, bool from_field_is_protected,
- TRAPS) const;
-
- public:
- static bool resolve_and_check_assignability(InstanceKlass* klass, Symbol* name,
+ static bool resolve_and_check_assignability(InstanceKlass* current_klass, Symbol* target_name,
Symbol* from_name, bool from_field_is_protected,
bool from_is_array, bool from_is_object,
+ TRAPS) {
+ return resolve_and_check_assignability(current_klass, target_name, from_name, from_field_is_protected,
+ from_is_array, from_is_object, nullptr, THREAD);
+ }
+
+ private:
+ static bool resolve_and_check_assignability(InstanceKlass* current_klass, Symbol* target_name,
+ Symbol* from_name, bool from_field_is_protected,
+ bool from_is_array, bool from_is_object,
+ bool* target_is_interface,
TRAPS);
+
};
#endif // SHARE_CLASSFILE_VERIFICATIONTYPE_HPP
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index 98c05909b82..0f1468f0309 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -2891,26 +2891,43 @@ void ClassVerifier::verify_invoke_instructions(
"Illegal call to internal method");
return;
}
- } else if (opcode == Bytecodes::_invokespecial
- && !is_same_or_direct_interface(current_class(), current_type(), ref_class_type)
- && !ref_class_type.equals(VerificationType::reference_type(
- current_class()->super()->name()))) {
- bool subtype = false;
- bool have_imr_indirect = cp->tag_at(index).value() == JVM_CONSTANT_InterfaceMethodref;
- subtype = ref_class_type.is_assignable_from(
- current_type(), this, false, CHECK_VERIFY(this));
- if (!subtype) {
- verify_error(ErrorContext::bad_code(bci),
- "Bad invokespecial instruction: "
- "current class isn't assignable to reference class.");
- return;
- } else if (have_imr_indirect) {
- verify_error(ErrorContext::bad_code(bci),
- "Bad invokespecial instruction: "
- "interface method reference is in an indirect superinterface.");
- return;
- }
+ }
+ // invokespecial, when not , must be to a method in the current class, a direct superinterface,
+ // or any superclass (including Object).
+ else if (opcode == Bytecodes::_invokespecial
+ && !is_same_or_direct_interface(current_class(), current_type(), ref_class_type)
+ && !ref_class_type.equals(VerificationType::reference_type(current_class()->super()->name()))) {
+ // We know it is not current class, direct superinterface or immediate superclass. That means it
+ // could be:
+ // - a totally unrelated class or interface
+ // - an indirect superinterface
+ // - an indirect superclass (including Object)
+ // We use the assignability test to see if it is a superclass, or else an interface, and keep track
+ // of the latter. Note that subtype can be true if we are dealing with an interface that is not actually
+ // implemented as assignability treats all interfaces as Object.
+
+ bool is_interface = false; // This can only be set true if the assignability check will return true
+ // and we loaded the class. For any other "true" returns (e.g. same class
+ // or Object) we either can't get here (same class already excluded above)
+ // or we know it is not an interface (i.e. Object).
+ bool subtype = ref_class_type.is_reference_assignable_from(current_type(), this, false,
+ &is_interface, CHECK_VERIFY(this));
+ if (!subtype) { // Totally unrelated class
+ verify_error(ErrorContext::bad_code(bci),
+ "Bad invokespecial instruction: "
+ "current class isn't assignable to reference class.");
+ return;
+ } else {
+ // Indirect superclass (including Object), indirect interface, or unrelated interface.
+ // Any interface use is an error.
+ if (is_interface) {
+ verify_error(ErrorContext::bad_code(bci),
+ "Bad invokespecial instruction: "
+ "interface method to invoke is not in a direct superinterface.");
+ return;
+ }
+ }
}
// Get the verification types for the method's arguments.
diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp
index 6fd5c07e137..baa945cdddf 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.cpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.cpp
@@ -289,8 +289,6 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
case vmIntrinsics::_dsin:
case vmIntrinsics::_dcos:
case vmIntrinsics::_dtan:
- case vmIntrinsics::_dtanh:
- case vmIntrinsics::_dcbrt:
case vmIntrinsics::_dlog:
case vmIntrinsics::_dexp:
case vmIntrinsics::_dpow:
@@ -316,6 +314,13 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
case vmIntrinsics::_fmaF:
if (!InlineMathNatives || !UseFMA) return true;
break;
+ case vmIntrinsics::_dtanh:
+ case vmIntrinsics::_dcbrt:
+ if (!InlineMathNatives || !InlineIntrinsics) return true;
+#if defined(AMD64) && (defined(COMPILER1) || defined(COMPILER2))
+ if (!UseLibmIntrinsic) return true;
+#endif
+ break;
case vmIntrinsics::_floatToFloat16:
case vmIntrinsics::_float16ToFloat:
if (!InlineIntrinsics) return true;
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index f9f6bd07254..dc9ce61627b 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -742,6 +742,12 @@ class SerializeClosure;
template(jdk_internal_vm_ThreadDumper, "jdk/internal/vm/ThreadDumper") \
template(dumpThreads_name, "dumpThreads") \
template(dumpThreadsToJson_name, "dumpThreadsToJson") \
+ template(jdk_internal_vm_ThreadSnapshot, "jdk/internal/vm/ThreadSnapshot") \
+ template(jdk_internal_vm_ThreadLock, "jdk/internal/vm/ThreadSnapshot$ThreadLock") \
+ template(jdk_internal_vm_ThreadLock_signature, "Ljdk/internal/vm/ThreadSnapshot$ThreadLock;") \
+ template(jdk_internal_vm_ThreadLock_array, "[Ljdk/internal/vm/ThreadSnapshot$ThreadLock;") \
+ template(java_lang_StackTraceElement_of_name, "of") \
+ template(java_lang_StackTraceElement_of_signature, "([Ljava/lang/StackTraceElement;)[Ljava/lang/StackTraceElement;") \
\
/* jcmd Thread.vthread_scheduler and Thread.vthread_pollers */ \
template(jdk_internal_vm_JcmdVThreadCommands, "jdk/internal/vm/JcmdVThreadCommands") \
diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp
index 674d71d1bfc..d0692ee678b 100644
--- a/src/hotspot/share/code/aotCodeCache.cpp
+++ b/src/hotspot/share/code/aotCodeCache.cpp
@@ -460,18 +460,9 @@ AOTCodeCache* AOTCodeCache::open_for_dump() {
}
void copy_bytes(const char* from, address to, uint size) {
- assert(size > 0, "sanity");
- bool by_words = true;
- if ((size > 2 * HeapWordSize) && (((intptr_t)from | (intptr_t)to) & (HeapWordSize - 1)) == 0) {
- // Use wordwise copies if possible:
- Copy::disjoint_words((HeapWord*)from,
- (HeapWord*)to,
- ((size_t)size + HeapWordSize-1) / HeapWordSize);
- } else {
- by_words = false;
- Copy::conjoint_jbytes(from, to, (size_t)size);
- }
- log_trace(aot, codecache)("Copied %d bytes as %s from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, (by_words ? "HeapWord" : "bytes"), p2i(from), p2i(to));
+ assert((int)size > 0, "sanity");
+ memcpy(to, from, size);
+ log_trace(aot, codecache)("Copied %d bytes from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, p2i(from), p2i(to));
}
AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry) {
@@ -915,26 +906,22 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name, int entry_offset_co
oop_maps = read_oop_map_set();
}
-#ifndef PRODUCT
- AsmRemarks asm_remarks;
- read_asm_remarks(asm_remarks);
- DbgStrings dbg_strings;
- read_dbg_strings(dbg_strings);
-#endif // PRODUCT
-
CodeBlob* code_blob = CodeBlob::create(archived_blob,
stored_name,
reloc_data,
oop_maps
-#ifndef PRODUCT
- , asm_remarks
- , dbg_strings
-#endif
);
if (code_blob == nullptr) { // no space left in CodeCache
return nullptr;
}
+#ifndef PRODUCT
+ code_blob->asm_remarks().init();
+ read_asm_remarks(code_blob->asm_remarks());
+ code_blob->dbg_strings().init();
+ read_dbg_strings(code_blob->dbg_strings());
+#endif // PRODUCT
+
fix_relocations(code_blob);
// Read entries offsets
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index 185b6c06598..5bb37c198d0 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -281,10 +281,6 @@ CodeBlob* CodeBlob::create(CodeBlob* archived_blob,
const char* name,
address archived_reloc_data,
ImmutableOopMapSet* archived_oop_maps
-#ifndef PRODUCT
- , AsmRemarks& archived_asm_remarks
- , DbgStrings& archived_dbg_strings
-#endif // PRODUCT
)
{
ThreadInVMfromUnknown __tiv; // get to VM state in case we block on CodeCache_lock
@@ -303,13 +299,6 @@ CodeBlob* CodeBlob::create(CodeBlob* archived_blob,
archived_oop_maps);
assert(blob != nullptr, "sanity check");
-#ifndef PRODUCT
- blob->use_remarks(archived_asm_remarks);
- archived_asm_remarks.clear();
- blob->use_strings(archived_dbg_strings);
- archived_dbg_strings.clear();
-#endif // PRODUCT
-
// Flush the code block
ICache::invalidate_range(blob->code_begin(), blob->code_size());
CodeCache::commit(blob); // Count adapters
diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp
index f813752e01e..f1920a829fc 100644
--- a/src/hotspot/share/code/codeBlob.hpp
+++ b/src/hotspot/share/code/codeBlob.hpp
@@ -318,12 +318,7 @@ public:
static CodeBlob* create(CodeBlob* archived_blob,
const char* name,
address archived_reloc_data,
- ImmutableOopMapSet* archived_oop_maps
-#ifndef PRODUCT
- , AsmRemarks& archived_asm_remarks
- , DbgStrings& archived_dbg_strings
-#endif // PRODUCT
- );
+ ImmutableOopMapSet* archived_oop_maps);
};
//----------------------------------------------------------------------------------------------------
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index 902d4345622..3a5da1d7cd2 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -1361,7 +1361,7 @@ void CodeCache::make_marked_nmethods_deoptimized() {
while(iter.next()) {
nmethod* nm = iter.method();
if (nm->is_marked_for_deoptimization() && !nm->has_been_deoptimized() && nm->can_be_deoptimized()) {
- nm->make_not_entrant("marked for deoptimization");
+ nm->make_not_entrant(nmethod::ChangeReason::marked_for_deoptimization);
nm->make_deoptimized();
}
}
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index 83afc02cdfc..acebaae6ba4 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -788,6 +788,8 @@ class CheckClass : public MetadataClosure {
klass = ((Method*)md)->method_holder();
} else if (md->is_methodData()) {
klass = ((MethodData*)md)->method()->method_holder();
+ } else if (md->is_methodCounters()) {
+ klass = ((MethodCounters*)md)->method()->method_holder();
} else {
md->print();
ShouldNotReachHere();
@@ -1973,14 +1975,12 @@ void nmethod::invalidate_osr_method() {
}
}
-void nmethod::log_state_change(const char* reason) const {
- assert(reason != nullptr, "Must provide a reason");
-
+void nmethod::log_state_change(ChangeReason change_reason) const {
if (LogCompilation) {
if (xtty != nullptr) {
ttyLocker ttyl; // keep the following output all in one block
xtty->begin_elem("make_not_entrant thread='%zu' reason='%s'",
- os::current_thread_id(), reason);
+ os::current_thread_id(), change_reason_to_string(change_reason));
log_identity(xtty);
xtty->stamp();
xtty->end_elem();
@@ -1989,7 +1989,7 @@ void nmethod::log_state_change(const char* reason) const {
ResourceMark rm;
stringStream ss(NEW_RESOURCE_ARRAY(char, 256), 256);
- ss.print("made not entrant: %s", reason);
+ ss.print("made not entrant: %s", change_reason_to_string(change_reason));
CompileTask::print_ul(this, ss.freeze());
if (PrintCompilation) {
@@ -2004,9 +2004,7 @@ void nmethod::unlink_from_method() {
}
// Invalidate code
-bool nmethod::make_not_entrant(const char* reason) {
- assert(reason != nullptr, "Must provide a reason");
-
+bool nmethod::make_not_entrant(ChangeReason change_reason) {
// This can be called while the system is already at a safepoint which is ok
NoSafepointVerifier nsv;
@@ -2075,7 +2073,7 @@ bool nmethod::make_not_entrant(const char* reason) {
assert(success, "Transition can't fail");
// Log the transition once
- log_state_change(reason);
+ log_state_change(change_reason);
// Remove nmethod from method.
unlink_from_method();
@@ -2143,10 +2141,19 @@ void nmethod::purge(bool unregister_nmethod) {
// completely deallocate this method
Events::log_nmethod_flush(Thread::current(), "flushing %s nmethod " INTPTR_FORMAT, is_osr_method() ? "osr" : "", p2i(this));
- log_debug(codecache)("*flushing %s nmethod %3d/" INTPTR_FORMAT ". Live blobs:" UINT32_FORMAT
- "/Free CodeCache:%zuKb",
- is_osr_method() ? "osr" : "",_compile_id, p2i(this), CodeCache::blob_count(),
- CodeCache::unallocated_capacity(CodeCache::get_code_blob_type(this))/1024);
+
+ LogTarget(Debug, codecache) lt;
+ if (lt.is_enabled()) {
+ ResourceMark rm;
+ LogStream ls(lt);
+ const char* method_name = method()->name()->as_C_string();
+ const size_t codecache_capacity = CodeCache::capacity()/1024;
+ const size_t codecache_free_space = CodeCache::unallocated_capacity(CodeCache::get_code_blob_type(this))/1024;
+ ls.print("Flushing nmethod %6d/" INTPTR_FORMAT ", level=%d, osr=%d, cold=%d, epoch=" UINT64_FORMAT ", cold_count=" UINT64_FORMAT ". "
+ "Cache capacity: %zuKb, free space: %zuKb. method %s (%s)",
+ _compile_id, p2i(this), _comp_level, is_osr_method(), is_cold(), _gc_epoch, CodeCache::cold_gc_count(),
+ codecache_capacity, codecache_free_space, method_name, compiler_name());
+ }
// We need to deallocate any ExceptionCache data.
// Note that we do not need to grab the nmethod lock for this, it
diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp
index 2ce6e5cd361..7453bdfa0ef 100644
--- a/src/hotspot/share/code/nmethod.hpp
+++ b/src/hotspot/share/code/nmethod.hpp
@@ -471,6 +471,85 @@ class nmethod : public CodeBlob {
void oops_do_set_strong_done(nmethod* old_head);
public:
+ enum class ChangeReason : u1 {
+ C1_codepatch,
+ C1_deoptimize,
+ C1_deoptimize_for_patching,
+ C1_predicate_failed_trap,
+ CI_replay,
+ JVMCI_invalidate_nmethod,
+ JVMCI_invalidate_nmethod_mirror,
+ JVMCI_materialize_virtual_object,
+ JVMCI_new_installation,
+ JVMCI_register_method,
+ JVMCI_replacing_with_new_code,
+ JVMCI_reprofile,
+ marked_for_deoptimization,
+ missing_exception_handler,
+ not_used,
+ OSR_invalidation_back_branch,
+ OSR_invalidation_for_compiling_with_C1,
+ OSR_invalidation_of_lower_level,
+ set_native_function,
+ uncommon_trap,
+ whitebox_deoptimization,
+ zombie,
+ };
+
+
+ static const char* change_reason_to_string(ChangeReason change_reason) {
+ switch (change_reason) {
+ case ChangeReason::C1_codepatch:
+ return "C1 code patch";
+ case ChangeReason::C1_deoptimize:
+ return "C1 deoptimized";
+ case ChangeReason::C1_deoptimize_for_patching:
+ return "C1 deoptimize for patching";
+ case ChangeReason::C1_predicate_failed_trap:
+ return "C1 predicate failed trap";
+ case ChangeReason::CI_replay:
+ return "CI replay";
+ case ChangeReason::JVMCI_invalidate_nmethod:
+ return "JVMCI invalidate nmethod";
+ case ChangeReason::JVMCI_invalidate_nmethod_mirror:
+ return "JVMCI invalidate nmethod mirror";
+ case ChangeReason::JVMCI_materialize_virtual_object:
+ return "JVMCI materialize virtual object";
+ case ChangeReason::JVMCI_new_installation:
+ return "JVMCI new installation";
+ case ChangeReason::JVMCI_register_method:
+ return "JVMCI register method";
+ case ChangeReason::JVMCI_replacing_with_new_code:
+ return "JVMCI replacing with new code";
+ case ChangeReason::JVMCI_reprofile:
+ return "JVMCI reprofile";
+ case ChangeReason::marked_for_deoptimization:
+ return "marked for deoptimization";
+ case ChangeReason::missing_exception_handler:
+ return "missing exception handler";
+ case ChangeReason::not_used:
+ return "not used";
+ case ChangeReason::OSR_invalidation_back_branch:
+ return "OSR invalidation back branch";
+ case ChangeReason::OSR_invalidation_for_compiling_with_C1:
+ return "OSR invalidation for compiling with C1";
+ case ChangeReason::OSR_invalidation_of_lower_level:
+ return "OSR invalidation of lower level";
+ case ChangeReason::set_native_function:
+ return "set native function";
+ case ChangeReason::uncommon_trap:
+ return "uncommon trap";
+ case ChangeReason::whitebox_deoptimization:
+ return "whitebox deoptimization";
+ case ChangeReason::zombie:
+ return "zombie";
+ default: {
+ assert(false, "Unhandled reason");
+ return "Unknown";
+ }
+ }
+ }
+
// create nmethod with entry_bci
static nmethod* new_nmethod(const methodHandle& method,
int compile_id,
@@ -633,8 +712,8 @@ public:
// alive. It is used when an uncommon trap happens. Returns true
// if this thread changed the state of the nmethod or false if
// another thread performed the transition.
- bool make_not_entrant(const char* reason);
- bool make_not_used() { return make_not_entrant("not used"); }
+ bool make_not_entrant(ChangeReason change_reason);
+ bool make_not_used() { return make_not_entrant(ChangeReason::not_used); }
bool is_marked_for_deoptimization() const { return deoptimization_status() != not_marked; }
bool has_been_deoptimized() const { return deoptimization_status() == deoptimize_done; }
@@ -947,7 +1026,7 @@ public:
// Logging
void log_identity(xmlStream* log) const;
void log_new_nmethod() const;
- void log_state_change(const char* reason) const;
+ void log_state_change(ChangeReason change_reason) const;
// Prints block-level comments, including nmethod specific block labels:
void print_nmethod_labels(outputStream* stream, address block_begin, bool print_section_labels=true) const;
diff --git a/src/hotspot/share/compiler/compilationPolicy.cpp b/src/hotspot/share/compiler/compilationPolicy.cpp
index d95debeda44..bab437eaade 100644
--- a/src/hotspot/share/compiler/compilationPolicy.cpp
+++ b/src/hotspot/share/compiler/compilationPolicy.cpp
@@ -550,6 +550,7 @@ void CompilationPolicy::initialize() {
int count = CICompilerCount;
bool c1_only = CompilerConfig::is_c1_only();
bool c2_only = CompilerConfig::is_c2_or_jvmci_compiler_only();
+ int min_count = (c1_only || c2_only) ? 1 : 2;
#ifdef _LP64
// Turn on ergonomic compiler count selection
@@ -560,7 +561,7 @@ void CompilationPolicy::initialize() {
// Simple log n seems to grow too slowly for tiered, try something faster: log n * log log n
int log_cpu = log2i(os::active_processor_count());
int loglog_cpu = log2i(MAX2(log_cpu, 1));
- count = MAX2(log_cpu * loglog_cpu * 3 / 2, 2);
+ count = MAX2(log_cpu * loglog_cpu * 3 / 2, min_count);
// Make sure there is enough space in the code cache to hold all the compiler buffers
size_t c1_size = 0;
#ifdef COMPILER1
@@ -574,7 +575,7 @@ void CompilationPolicy::initialize() {
int max_count = (ReservedCodeCacheSize - (CodeCacheMinimumUseSpace DEBUG_ONLY(* 3))) / (int)buffer_size;
if (count > max_count) {
// Lower the compiler count such that all buffers fit into the code cache
- count = MAX2(max_count, c1_only ? 1 : 2);
+ count = MAX2(max_count, min_count);
}
FLAG_SET_ERGO(CICompilerCount, count);
}
@@ -593,9 +594,10 @@ void CompilationPolicy::initialize() {
#endif
if (c1_only) {
- // No C2 compiler thread required
+ // No C2 compiler threads are needed
set_c1_count(count);
} else if (c2_only) {
+ // No C1 compiler threads are needed
set_c2_count(count);
} else {
#if INCLUDE_JVMCI
@@ -613,6 +615,9 @@ void CompilationPolicy::initialize() {
}
assert(count == c1_count() + c2_count(), "inconsistent compiler thread count");
set_increase_threshold_at_ratio();
+ } else {
+ // Interpreter mode creates no compilers
+ FLAG_SET_ERGO(CICompilerCount, 0);
}
set_start_time(nanos_to_millis(os::javaTimeNanos()));
}
@@ -919,7 +924,7 @@ void CompilationPolicy::compile(const methodHandle& mh, int bci, CompLevel level
nmethod* osr_nm = mh->lookup_osr_nmethod_for(bci, CompLevel_simple, false);
if (osr_nm != nullptr && osr_nm->comp_level() > CompLevel_simple) {
// Invalidate the existing OSR nmethod so that a compile at CompLevel_simple is permitted.
- osr_nm->make_not_entrant("OSR invalidation for compiling with C1");
+ osr_nm->make_not_entrant(nmethod::ChangeReason::OSR_invalidation_for_compiling_with_C1);
}
compile(mh, bci, CompLevel_simple, THREAD);
}
@@ -1511,7 +1516,7 @@ void CompilationPolicy::method_back_branch_event(const methodHandle& mh, const m
int osr_bci = nm->is_osr_method() ? nm->osr_entry_bci() : InvocationEntryBci;
print_event(MAKE_NOT_ENTRANT, mh(), mh(), osr_bci, level);
}
- nm->make_not_entrant("OSR invalidation, back branch");
+ nm->make_not_entrant(nmethod::ChangeReason::OSR_invalidation_back_branch);
}
}
// Fix up next_level if necessary to avoid deopts
diff --git a/src/hotspot/share/gc/parallel/parallelArguments.cpp b/src/hotspot/share/gc/parallel/parallelArguments.cpp
index 2cddbafd871..cae6f9a93d5 100644
--- a/src/hotspot/share/gc/parallel/parallelArguments.cpp
+++ b/src/hotspot/share/gc/parallel/parallelArguments.cpp
@@ -98,15 +98,15 @@ void ParallelArguments::initialize() {
FullGCForwarding::initialize_flags(heap_reserved_size_bytes());
}
-// The alignment used for boundary between young gen and old gen
-static size_t default_gen_alignment() {
+// The alignment used for spaces in young gen and old gen
+static size_t default_space_alignment() {
return 64 * K * HeapWordSize;
}
void ParallelArguments::initialize_alignments() {
// Initialize card size before initializing alignments
CardTable::initialize_card_size();
- SpaceAlignment = GenAlignment = default_gen_alignment();
+ SpaceAlignment = default_space_alignment();
HeapAlignment = compute_heap_alignment();
}
@@ -123,9 +123,8 @@ void ParallelArguments::initialize_heap_flags_and_sizes() {
// Can a page size be something else than a power of two?
assert(is_power_of_2((intptr_t)page_sz), "must be a power of 2");
- size_t new_alignment = align_up(page_sz, GenAlignment);
- if (new_alignment != GenAlignment) {
- GenAlignment = new_alignment;
+ size_t new_alignment = align_up(page_sz, SpaceAlignment);
+ if (new_alignment != SpaceAlignment) {
SpaceAlignment = new_alignment;
// Redo everything from the start
initialize_heap_flags_and_sizes_one_pass();
diff --git a/src/hotspot/share/gc/parallel/parallelInitLogger.cpp b/src/hotspot/share/gc/parallel/parallelInitLogger.cpp
index 1d9c9f840ba..d7eb265d113 100644
--- a/src/hotspot/share/gc/parallel/parallelInitLogger.cpp
+++ b/src/hotspot/share/gc/parallel/parallelInitLogger.cpp
@@ -29,10 +29,8 @@
void ParallelInitLogger::print_heap() {
log_info_p(gc, init)("Alignments:"
" Space " EXACTFMT ","
- " Generation " EXACTFMT ","
" Heap " EXACTFMT,
EXACTFMTARGS(SpaceAlignment),
- EXACTFMTARGS(GenAlignment),
EXACTFMTARGS(HeapAlignment));
GCInitLogger::print_heap();
}
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
index 0235864992f..139a5fa52f1 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
@@ -69,8 +69,8 @@ jint ParallelScavengeHeap::initialize() {
initialize_reserved_region(heap_rs);
// Layout the reserved space for the generations.
- ReservedSpace old_rs = heap_rs.first_part(MaxOldSize, GenAlignment);
- ReservedSpace young_rs = heap_rs.last_part(MaxOldSize, GenAlignment);
+ ReservedSpace old_rs = heap_rs.first_part(MaxOldSize, SpaceAlignment);
+ ReservedSpace young_rs = heap_rs.last_part(MaxOldSize, SpaceAlignment);
assert(young_rs.size() == MaxNewSize, "Didn't reserve all of the heap");
PSCardTable* card_table = new PSCardTable(_reserved);
@@ -107,7 +107,7 @@ jint ParallelScavengeHeap::initialize() {
new PSAdaptiveSizePolicy(eden_capacity,
initial_promo_size,
young_gen()->to_space()->capacity_in_bytes(),
- GenAlignment,
+ SpaceAlignment,
max_gc_pause_sec,
GCTimeRatio
);
diff --git a/src/hotspot/share/gc/parallel/psOldGen.cpp b/src/hotspot/share/gc/parallel/psOldGen.cpp
index 0d1c3b60672..bc25683e00f 100644
--- a/src/hotspot/share/gc/parallel/psOldGen.cpp
+++ b/src/hotspot/share/gc/parallel/psOldGen.cpp
@@ -41,7 +41,7 @@ PSOldGen::PSOldGen(ReservedSpace rs, size_t initial_size, size_t min_size,
_min_gen_size(min_size),
_max_gen_size(max_size)
{
- initialize(rs, initial_size, GenAlignment);
+ initialize(rs, initial_size, SpaceAlignment);
}
void PSOldGen::initialize(ReservedSpace rs, size_t initial_size, size_t alignment) {
diff --git a/src/hotspot/share/gc/parallel/psYoungGen.cpp b/src/hotspot/share/gc/parallel/psYoungGen.cpp
index f9cb067411e..bf77a0dde93 100644
--- a/src/hotspot/share/gc/parallel/psYoungGen.cpp
+++ b/src/hotspot/share/gc/parallel/psYoungGen.cpp
@@ -47,7 +47,7 @@ PSYoungGen::PSYoungGen(ReservedSpace rs, size_t initial_size, size_t min_size, s
_from_counters(nullptr),
_to_counters(nullptr)
{
- initialize(rs, initial_size, GenAlignment);
+ initialize(rs, initial_size, SpaceAlignment);
}
void PSYoungGen::initialize_virtual_space(ReservedSpace rs,
@@ -746,7 +746,7 @@ size_t PSYoungGen::available_to_live() {
}
size_t delta_in_bytes = unused_committed + delta_in_survivor;
- delta_in_bytes = align_down(delta_in_bytes, GenAlignment);
+ delta_in_bytes = align_down(delta_in_bytes, SpaceAlignment);
return delta_in_bytes;
}
diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp
index 27d94593765..fbbaabf618a 100644
--- a/src/hotspot/share/gc/serial/serialHeap.cpp
+++ b/src/hotspot/share/gc/serial/serialHeap.cpp
@@ -188,8 +188,8 @@ jint SerialHeap::initialize() {
initialize_reserved_region(heap_rs);
- ReservedSpace young_rs = heap_rs.first_part(MaxNewSize, GenAlignment);
- ReservedSpace old_rs = heap_rs.last_part(MaxNewSize, GenAlignment);
+ ReservedSpace young_rs = heap_rs.first_part(MaxNewSize, SpaceAlignment);
+ ReservedSpace old_rs = heap_rs.last_part(MaxNewSize, SpaceAlignment);
_rem_set = new CardTableRS(_reserved);
_rem_set->initialize(young_rs.base(), old_rs.base());
diff --git a/src/hotspot/share/gc/shared/gcArguments.hpp b/src/hotspot/share/gc/shared/gcArguments.hpp
index 40b612d75a6..fff41e85d8c 100644
--- a/src/hotspot/share/gc/shared/gcArguments.hpp
+++ b/src/hotspot/share/gc/shared/gcArguments.hpp
@@ -35,7 +35,7 @@ extern size_t SpaceAlignment;
class GCArguments {
protected:
- // Initialize HeapAlignment, SpaceAlignment, and extra alignments (E.g. GenAlignment)
+ // Initialize HeapAlignment, SpaceAlignment
virtual void initialize_alignments() = 0;
virtual void initialize_heap_flags_and_sizes();
virtual void initialize_size_info();
diff --git a/src/hotspot/share/gc/shared/genArguments.cpp b/src/hotspot/share/gc/shared/genArguments.cpp
index 90617b1675f..9618c515b7d 100644
--- a/src/hotspot/share/gc/shared/genArguments.cpp
+++ b/src/hotspot/share/gc/shared/genArguments.cpp
@@ -42,17 +42,15 @@ size_t MaxOldSize = 0;
// See more in JDK-8346005
size_t OldSize = ScaleForWordSize(4*M);
-size_t GenAlignment = 0;
-
size_t GenArguments::conservative_max_heap_alignment() { return (size_t)Generation::GenGrain; }
static size_t young_gen_size_lower_bound() {
// The young generation must be aligned and have room for eden + two survivors
- return align_up(3 * SpaceAlignment, GenAlignment);
+ return 3 * SpaceAlignment;
}
static size_t old_gen_size_lower_bound() {
- return align_up(SpaceAlignment, GenAlignment);
+ return SpaceAlignment;
}
size_t GenArguments::scale_by_NewRatio_aligned(size_t base_size, size_t alignment) {
@@ -69,23 +67,20 @@ static size_t bound_minus_alignment(size_t desired_size,
void GenArguments::initialize_alignments() {
// Initialize card size before initializing alignments
CardTable::initialize_card_size();
- SpaceAlignment = GenAlignment = (size_t)Generation::GenGrain;
+ SpaceAlignment = (size_t)Generation::GenGrain;
HeapAlignment = compute_heap_alignment();
}
void GenArguments::initialize_heap_flags_and_sizes() {
GCArguments::initialize_heap_flags_and_sizes();
- assert(GenAlignment != 0, "Generation alignment not set up properly");
- assert(HeapAlignment >= GenAlignment,
- "HeapAlignment: %zu less than GenAlignment: %zu",
- HeapAlignment, GenAlignment);
- assert(GenAlignment % SpaceAlignment == 0,
- "GenAlignment: %zu not aligned by SpaceAlignment: %zu",
- GenAlignment, SpaceAlignment);
- assert(HeapAlignment % GenAlignment == 0,
- "HeapAlignment: %zu not aligned by GenAlignment: %zu",
- HeapAlignment, GenAlignment);
+ assert(SpaceAlignment != 0, "Generation alignment not set up properly");
+ assert(HeapAlignment >= SpaceAlignment,
+ "HeapAlignment: %zu less than SpaceAlignment: %zu",
+ HeapAlignment, SpaceAlignment);
+ assert(HeapAlignment % SpaceAlignment == 0,
+ "HeapAlignment: %zu not aligned by SpaceAlignment: %zu",
+ HeapAlignment, SpaceAlignment);
// All generational heaps have a young gen; handle those flags here
@@ -106,7 +101,7 @@ void GenArguments::initialize_heap_flags_and_sizes() {
// Make sure NewSize allows an old generation to fit even if set on the command line
if (FLAG_IS_CMDLINE(NewSize) && NewSize >= InitialHeapSize) {
- size_t revised_new_size = bound_minus_alignment(NewSize, InitialHeapSize, GenAlignment);
+ size_t revised_new_size = bound_minus_alignment(NewSize, InitialHeapSize, SpaceAlignment);
log_warning(gc, ergo)("NewSize (%zuk) is equal to or greater than initial heap size (%zuk). A new "
"NewSize of %zuk will be used to accomodate an old generation.",
NewSize/K, InitialHeapSize/K, revised_new_size/K);
@@ -115,8 +110,8 @@ void GenArguments::initialize_heap_flags_and_sizes() {
// Now take the actual NewSize into account. We will silently increase NewSize
// if the user specified a smaller or unaligned value.
- size_t bounded_new_size = bound_minus_alignment(NewSize, MaxHeapSize, GenAlignment);
- bounded_new_size = MAX2(smallest_new_size, align_down(bounded_new_size, GenAlignment));
+ size_t bounded_new_size = bound_minus_alignment(NewSize, MaxHeapSize, SpaceAlignment);
+ bounded_new_size = MAX2(smallest_new_size, align_down(bounded_new_size, SpaceAlignment));
if (bounded_new_size != NewSize) {
FLAG_SET_ERGO(NewSize, bounded_new_size);
}
@@ -125,7 +120,7 @@ void GenArguments::initialize_heap_flags_and_sizes() {
if (!FLAG_IS_DEFAULT(MaxNewSize)) {
if (MaxNewSize >= MaxHeapSize) {
// Make sure there is room for an old generation
- size_t smaller_max_new_size = MaxHeapSize - GenAlignment;
+ size_t smaller_max_new_size = MaxHeapSize - SpaceAlignment;
if (FLAG_IS_CMDLINE(MaxNewSize)) {
log_warning(gc, ergo)("MaxNewSize (%zuk) is equal to or greater than the entire "
"heap (%zuk). A new max generation size of %zuk will be used.",
@@ -137,8 +132,8 @@ void GenArguments::initialize_heap_flags_and_sizes() {
}
} else if (MaxNewSize < NewSize) {
FLAG_SET_ERGO(MaxNewSize, NewSize);
- } else if (!is_aligned(MaxNewSize, GenAlignment)) {
- FLAG_SET_ERGO(MaxNewSize, align_down(MaxNewSize, GenAlignment));
+ } else if (!is_aligned(MaxNewSize, SpaceAlignment)) {
+ FLAG_SET_ERGO(MaxNewSize, align_down(MaxNewSize, SpaceAlignment));
}
}
@@ -166,13 +161,13 @@ void GenArguments::initialize_heap_flags_and_sizes() {
// exceed it. Adjust New/OldSize as necessary.
size_t calculated_size = NewSize + OldSize;
double shrink_factor = (double) MaxHeapSize / calculated_size;
- size_t smaller_new_size = align_down((size_t)(NewSize * shrink_factor), GenAlignment);
+ size_t smaller_new_size = align_down((size_t)(NewSize * shrink_factor), SpaceAlignment);
FLAG_SET_ERGO(NewSize, MAX2(young_gen_size_lower_bound(), smaller_new_size));
// OldSize is already aligned because above we aligned MaxHeapSize to
// HeapAlignment, and we just made sure that NewSize is aligned to
- // GenAlignment. In initialize_flags() we verified that HeapAlignment
- // is a multiple of GenAlignment.
+ // SpaceAlignment. In initialize_flags() we verified that HeapAlignment
+ // is a multiple of SpaceAlignment.
OldSize = MaxHeapSize - NewSize;
} else {
FLAG_SET_ERGO(MaxHeapSize, align_up(NewSize + OldSize, HeapAlignment));
@@ -200,7 +195,7 @@ void GenArguments::initialize_size_info() {
// Determine maximum size of the young generation.
if (FLAG_IS_DEFAULT(MaxNewSize)) {
- max_young_size = scale_by_NewRatio_aligned(MaxHeapSize, GenAlignment);
+ max_young_size = scale_by_NewRatio_aligned(MaxHeapSize, SpaceAlignment);
// Bound the maximum size by NewSize below (since it historically
// would have been NewSize and because the NewRatio calculation could
// yield a size that is too small) and bound it by MaxNewSize above.
@@ -229,18 +224,18 @@ void GenArguments::initialize_size_info() {
// If NewSize is set on the command line, we should use it as
// the initial size, but make sure it is within the heap bounds.
initial_young_size =
- MIN2(max_young_size, bound_minus_alignment(NewSize, InitialHeapSize, GenAlignment));
- MinNewSize = bound_minus_alignment(initial_young_size, MinHeapSize, GenAlignment);
+ MIN2(max_young_size, bound_minus_alignment(NewSize, InitialHeapSize, SpaceAlignment));
+ MinNewSize = bound_minus_alignment(initial_young_size, MinHeapSize, SpaceAlignment);
} else {
// For the case where NewSize is not set on the command line, use
// NewRatio to size the initial generation size. Use the current
// NewSize as the floor, because if NewRatio is overly large, the resulting
// size can be too small.
initial_young_size =
- clamp(scale_by_NewRatio_aligned(InitialHeapSize, GenAlignment), NewSize, max_young_size);
+ clamp(scale_by_NewRatio_aligned(InitialHeapSize, SpaceAlignment), NewSize, max_young_size);
// Derive MinNewSize from MinHeapSize
- MinNewSize = MIN2(scale_by_NewRatio_aligned(MinHeapSize, GenAlignment), initial_young_size);
+ MinNewSize = MIN2(scale_by_NewRatio_aligned(MinHeapSize, SpaceAlignment), initial_young_size);
}
}
@@ -252,7 +247,7 @@ void GenArguments::initialize_size_info() {
// The maximum old size can be determined from the maximum young
// and maximum heap size since no explicit flags exist
// for setting the old generation maximum.
- MaxOldSize = MAX2(MaxHeapSize - max_young_size, GenAlignment);
+ MaxOldSize = MAX2(MaxHeapSize - max_young_size, SpaceAlignment);
MinOldSize = MIN3(MaxOldSize,
InitialHeapSize - initial_young_size,
MinHeapSize - MinNewSize);
@@ -315,10 +310,10 @@ void GenArguments::assert_flags() {
assert(NewSize >= MinNewSize, "Ergonomics decided on a too small young gen size");
assert(NewSize <= MaxNewSize, "Ergonomics decided on incompatible initial and maximum young gen sizes");
assert(FLAG_IS_DEFAULT(MaxNewSize) || MaxNewSize < MaxHeapSize, "Ergonomics decided on incompatible maximum young gen and heap sizes");
- assert(NewSize % GenAlignment == 0, "NewSize alignment");
- assert(FLAG_IS_DEFAULT(MaxNewSize) || MaxNewSize % GenAlignment == 0, "MaxNewSize alignment");
+ assert(NewSize % SpaceAlignment == 0, "NewSize alignment");
+ assert(FLAG_IS_DEFAULT(MaxNewSize) || MaxNewSize % SpaceAlignment == 0, "MaxNewSize alignment");
assert(OldSize + NewSize <= MaxHeapSize, "Ergonomics decided on incompatible generation and heap sizes");
- assert(OldSize % GenAlignment == 0, "OldSize alignment");
+ assert(OldSize % SpaceAlignment == 0, "OldSize alignment");
}
void GenArguments::assert_size_info() {
@@ -327,19 +322,19 @@ void GenArguments::assert_size_info() {
assert(MaxNewSize < MaxHeapSize, "Ergonomics decided on incompatible maximum young and heap sizes");
assert(MinNewSize <= NewSize, "Ergonomics decided on incompatible minimum and initial young gen sizes");
assert(NewSize <= MaxNewSize, "Ergonomics decided on incompatible initial and maximum young gen sizes");
- assert(MinNewSize % GenAlignment == 0, "_min_young_size alignment");
- assert(NewSize % GenAlignment == 0, "_initial_young_size alignment");
- assert(MaxNewSize % GenAlignment == 0, "MaxNewSize alignment");
- assert(MinNewSize <= bound_minus_alignment(MinNewSize, MinHeapSize, GenAlignment),
+ assert(MinNewSize % SpaceAlignment == 0, "_min_young_size alignment");
+ assert(NewSize % SpaceAlignment == 0, "_initial_young_size alignment");
+ assert(MaxNewSize % SpaceAlignment == 0, "MaxNewSize alignment");
+ assert(MinNewSize <= bound_minus_alignment(MinNewSize, MinHeapSize, SpaceAlignment),
"Ergonomics made minimum young generation larger than minimum heap");
- assert(NewSize <= bound_minus_alignment(NewSize, InitialHeapSize, GenAlignment),
+ assert(NewSize <= bound_minus_alignment(NewSize, InitialHeapSize, SpaceAlignment),
"Ergonomics made initial young generation larger than initial heap");
- assert(MaxNewSize <= bound_minus_alignment(MaxNewSize, MaxHeapSize, GenAlignment),
+ assert(MaxNewSize <= bound_minus_alignment(MaxNewSize, MaxHeapSize, SpaceAlignment),
"Ergonomics made maximum young generation lager than maximum heap");
assert(MinOldSize <= OldSize, "Ergonomics decided on incompatible minimum and initial old gen sizes");
assert(OldSize <= MaxOldSize, "Ergonomics decided on incompatible initial and maximum old gen sizes");
- assert(MaxOldSize % GenAlignment == 0, "MaxOldSize alignment");
- assert(OldSize % GenAlignment == 0, "OldSize alignment");
+ assert(MaxOldSize % SpaceAlignment == 0, "MaxOldSize alignment");
+ assert(OldSize % SpaceAlignment == 0, "OldSize alignment");
assert(MaxHeapSize <= (MaxNewSize + MaxOldSize), "Total maximum heap sizes must be sum of generation maximum sizes");
assert(MinNewSize + MinOldSize <= MinHeapSize, "Minimum generation sizes exceed minimum heap size");
assert(NewSize + OldSize == InitialHeapSize, "Initial generation sizes should match initial heap size");
diff --git a/src/hotspot/share/gc/shared/genArguments.hpp b/src/hotspot/share/gc/shared/genArguments.hpp
index ae20c6126fc..80133bd1ec1 100644
--- a/src/hotspot/share/gc/shared/genArguments.hpp
+++ b/src/hotspot/share/gc/shared/genArguments.hpp
@@ -35,8 +35,6 @@ extern size_t MaxOldSize;
extern size_t OldSize;
-extern size_t GenAlignment;
-
class GenArguments : public GCArguments {
friend class TestGenCollectorPolicy; // Testing
private:
diff --git a/src/hotspot/share/gc/z/zAllocator.cpp b/src/hotspot/share/gc/z/zAllocator.cpp
index 22f3b6ba112..dd1ab3b712a 100644
--- a/src/hotspot/share/gc/z/zAllocator.cpp
+++ b/src/hotspot/share/gc/z/zAllocator.cpp
@@ -23,6 +23,7 @@
#include "gc/z/zAllocator.hpp"
#include "gc/z/zObjectAllocator.hpp"
+#include "gc/z/zPageAge.inline.hpp"
ZAllocatorEden* ZAllocator::_eden;
ZAllocatorForRelocation* ZAllocator::_relocation[ZAllocator::_relocation_allocators];
@@ -47,7 +48,7 @@ ZPageAge ZAllocatorForRelocation::install() {
for (uint i = 0; i < ZAllocator::_relocation_allocators; ++i) {
if (_relocation[i] == nullptr) {
_relocation[i] = this;
- return static_cast(i + 1);
+ return to_zpageage(i + 1);
}
}
diff --git a/src/hotspot/share/gc/z/zAllocator.hpp b/src/hotspot/share/gc/z/zAllocator.hpp
index 45a70888f9d..ec4c0ec1c9c 100644
--- a/src/hotspot/share/gc/z/zAllocator.hpp
+++ b/src/hotspot/share/gc/z/zAllocator.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,7 @@ class ZPage;
class ZAllocator {
public:
- static constexpr uint _relocation_allocators = static_cast(ZPageAge::old);
+ static constexpr uint _relocation_allocators = ZPageAgeCount - 1;
protected:
ZObjectAllocator _object_allocator;
diff --git a/src/hotspot/share/gc/z/zAllocator.inline.hpp b/src/hotspot/share/gc/z/zAllocator.inline.hpp
index ba558a1b0f3..ec64676dbf0 100644
--- a/src/hotspot/share/gc/z/zAllocator.inline.hpp
+++ b/src/hotspot/share/gc/z/zAllocator.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,13 +28,14 @@
#include "gc/z/zAddress.inline.hpp"
#include "gc/z/zHeap.hpp"
+#include "gc/z/zPageAge.inline.hpp"
inline ZAllocatorEden* ZAllocator::eden() {
return _eden;
}
inline ZAllocatorForRelocation* ZAllocator::relocation(ZPageAge page_age) {
- return _relocation[static_cast(page_age) - 1];
+ return _relocation[untype(page_age - 1)];
}
inline ZAllocatorForRelocation* ZAllocator::old() {
diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp
index 78860bbe4cb..534f0195c90 100644
--- a/src/hotspot/share/gc/z/zGeneration.cpp
+++ b/src/hotspot/share/gc/z/zGeneration.cpp
@@ -41,6 +41,7 @@
#include "gc/z/zHeap.inline.hpp"
#include "gc/z/zJNICritical.hpp"
#include "gc/z/zMark.inline.hpp"
+#include "gc/z/zPageAge.inline.hpp"
#include "gc/z/zPageAllocator.hpp"
#include "gc/z/zRelocationSet.inline.hpp"
#include "gc/z/zRelocationSetSelector.inline.hpp"
@@ -699,11 +700,10 @@ uint ZGenerationYoung::compute_tenuring_threshold(ZRelocationSetSelectorStats st
uint last_populated_age = 0;
size_t last_populated_live = 0;
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- const ZPageAge age = static_cast(i);
+ for (ZPageAge age : ZPageAgeRange()) {
const size_t young_live = stats.small(age).live() + stats.medium(age).live() + stats.large(age).live();
if (young_live > 0) {
- last_populated_age = i;
+ last_populated_age = untype(age);
last_populated_live = young_live;
if (young_live_last > 0) {
young_life_expectancy_sum += double(young_live) / double(young_live_last);
@@ -842,8 +842,8 @@ void ZGenerationYoung::mark_start() {
// Retire allocating pages
ZAllocator::eden()->retire_pages();
- for (ZPageAge i = ZPageAge::survivor1; i <= ZPageAge::survivor14; i = static_cast(static_cast(i) + 1)) {
- ZAllocator::relocation(i)->retire_pages();
+ for (ZPageAge age : ZPageAgeRangeSurvivor) {
+ ZAllocator::relocation(age)->retire_pages();
}
// Reset allocated/reclaimed/used statistics
@@ -948,6 +948,14 @@ void ZGenerationYoung::register_with_remset(ZPage* page) {
_remembered.register_found_old(page);
}
+ZRemembered* ZGenerationYoung::remembered() {
+ return &_remembered;
+}
+
+void ZGenerationYoung::remap_current_remset(ZRemsetTableIterator* iter) {
+ _remembered.remap_current(iter);
+}
+
ZGenerationTracer* ZGenerationYoung::jfr_tracer() {
return &_jfr_tracer;
}
@@ -1435,7 +1443,7 @@ typedef ClaimingCLDToOopClosure ZRemapCLDClosure;
class ZRemapYoungRootsTask : public ZTask {
private:
- ZGenerationPagesParallelIterator _old_pages_parallel_iterator;
+ ZRemsetTableIterator _remset_table_iterator;
ZRootsIteratorAllColored _roots_colored;
ZRootsIteratorAllUncolored _roots_uncolored;
@@ -1449,7 +1457,7 @@ private:
public:
ZRemapYoungRootsTask(ZPageTable* page_table, ZPageAllocator* page_allocator)
: ZTask("ZRemapYoungRootsTask"),
- _old_pages_parallel_iterator(page_table, ZGenerationId::old, page_allocator),
+ _remset_table_iterator(ZGeneration::young()->remembered(), false /* previous */),
_roots_colored(ZGenerationIdOptional::old),
_roots_uncolored(ZGenerationIdOptional::old),
_cl_colored(),
@@ -1472,11 +1480,8 @@ public:
{
ZStatTimerWorker timer(ZSubPhaseConcurrentRemapRememberedOld);
- _old_pages_parallel_iterator.do_pages([&](ZPage* page) {
- // Visit all object fields that potentially pointing into young generation
- page->oops_do_current_remembered(ZBarrier::load_barrier_on_oop_field);
- return true;
- });
+ // Visit all object fields that potentially pointing into young generation
+ ZGeneration::young()->remap_current_remset(&_remset_table_iterator);
}
}
};
diff --git a/src/hotspot/share/gc/z/zGeneration.hpp b/src/hotspot/share/gc/z/zGeneration.hpp
index 922efe5ef9e..13adc06b123 100644
--- a/src/hotspot/share/gc/z/zGeneration.hpp
+++ b/src/hotspot/share/gc/z/zGeneration.hpp
@@ -191,6 +191,7 @@ class ZGenerationYoung : public ZGeneration {
friend class VM_ZMarkStartYoung;
friend class VM_ZMarkStartYoungAndOld;
friend class VM_ZRelocateStartYoung;
+ friend class ZRemapYoungRootsTask;
friend class ZYoungTypeSetter;
private:
@@ -219,6 +220,8 @@ private:
void pause_relocate_start();
void concurrent_relocate();
+ ZRemembered* remembered();
+
public:
ZGenerationYoung(ZPageTable* page_table,
const ZForwardingTable* old_forwarding_table,
@@ -252,6 +255,9 @@ public:
// Register old pages with remembered set
void register_with_remset(ZPage* page);
+ // Remap the oops of the current remembered set
+ void remap_current_remset(ZRemsetTableIterator* iter);
+
// Serviceability
ZGenerationTracer* jfr_tracer();
diff --git a/src/hotspot/share/gc/z/zPageAge.hpp b/src/hotspot/share/gc/z/zPageAge.hpp
index b7b1688b830..3209790c008 100644
--- a/src/hotspot/share/gc/z/zPageAge.hpp
+++ b/src/hotspot/share/gc/z/zPageAge.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
#ifndef SHARE_GC_Z_ZPAGEAGE_HPP
#define SHARE_GC_Z_ZPAGEAGE_HPP
+#include "utilities/enumIterator.hpp"
#include "utilities/globalDefinitions.hpp"
enum class ZPageAge : uint8_t {
@@ -45,6 +46,19 @@ enum class ZPageAge : uint8_t {
old
};
-constexpr uint ZPageAgeMax = static_cast(ZPageAge::old);
+constexpr uint ZPageAgeCount = static_cast(ZPageAge::old) + 1;
+constexpr ZPageAge ZPageAgeLastPlusOne = static_cast(ZPageAgeCount);
+
+ENUMERATOR_RANGE(ZPageAge,
+ ZPageAge::eden,
+ ZPageAge::old);
+
+using ZPageAgeRange = EnumRange;
+
+constexpr ZPageAgeRange ZPageAgeRangeEden = ZPageAgeRange::create();
+constexpr ZPageAgeRange ZPageAgeRangeYoung = ZPageAgeRange::create();
+constexpr ZPageAgeRange ZPageAgeRangeSurvivor = ZPageAgeRange::create();
+constexpr ZPageAgeRange ZPageAgeRangeRelocation = ZPageAgeRange::create();
+constexpr ZPageAgeRange ZPageAgeRangeOld = ZPageAgeRange::create();
#endif // SHARE_GC_Z_ZPAGEAGE_HPP
diff --git a/src/hotspot/share/gc/z/zPageAge.inline.hpp b/src/hotspot/share/gc/z/zPageAge.inline.hpp
new file mode 100644
index 00000000000..0944caa69a6
--- /dev/null
+++ b/src/hotspot/share/gc/z/zPageAge.inline.hpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZPAGEAGE_INLINE_HPP
+#define SHARE_GC_Z_ZPAGEAGE_INLINE_HPP
+
+#include "gc/z/zPageAge.hpp"
+
+#include "utilities/checkedCast.hpp"
+
+#include
+
+inline uint untype(ZPageAge age) {
+ return static_cast(age);
+}
+
+inline ZPageAge to_zpageage(uint age) {
+ assert(age < ZPageAgeCount, "Invalid age");
+ return static_cast(age);
+}
+
+inline ZPageAge operator+(ZPageAge age, size_t size) {
+ const auto size_value = checked_cast>(size);
+ return to_zpageage(untype(age) + size_value);
+}
+
+inline ZPageAge operator-(ZPageAge age, size_t size) {
+ const auto size_value = checked_cast>(size);
+ return to_zpageage(untype(age) - size_value);
+}
+
+#endif // SHARE_GC_Z_ZPAGEAGE_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zRelocate.cpp b/src/hotspot/share/gc/z/zRelocate.cpp
index df2dd8a01cb..b45d8b70e72 100644
--- a/src/hotspot/share/gc/z/zRelocate.cpp
+++ b/src/hotspot/share/gc/z/zRelocate.cpp
@@ -488,11 +488,11 @@ public:
}
ZPage* shared(ZPageAge age) {
- return _shared[static_cast(age) - 1];
+ return _shared[untype(age - 1)];
}
void set_shared(ZPageAge age, ZPage* page) {
- _shared[static_cast(age) - 1] = page;
+ _shared[untype(age - 1)] = page;
}
ZPage* alloc_and_retire_target_page(ZForwarding* forwarding, ZPage* target) {
@@ -570,11 +570,11 @@ private:
ZPage* target(ZPageAge age) {
- return _target[static_cast(age) - 1];
+ return _target[untype(age - 1)];
}
void set_target(ZPageAge age, ZPage* page) {
- _target[static_cast(age) - 1] = page;
+ _target[untype(age - 1)] = page;
}
size_t object_alignment() const {
@@ -1232,12 +1232,12 @@ ZPageAge ZRelocate::compute_to_age(ZPageAge from_age) {
return ZPageAge::old;
}
- const uint age = static_cast(from_age);
+ const uint age = untype(from_age);
if (age >= ZGeneration::young()->tenuring_threshold()) {
return ZPageAge::old;
}
- return static_cast(age + 1);
+ return to_zpageage(age + 1);
}
class ZFlipAgePagesTask : public ZTask {
diff --git a/src/hotspot/share/gc/z/zRelocationSetSelector.cpp b/src/hotspot/share/gc/z/zRelocationSetSelector.cpp
index e9ef66a19dc..aac4fd51271 100644
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.cpp
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.cpp
@@ -25,6 +25,7 @@
#include "gc/z/zArray.inline.hpp"
#include "gc/z/zForwarding.inline.hpp"
#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageAge.inline.hpp"
#include "gc/z/zRelocationSetSelector.inline.hpp"
#include "jfr/jfrEvents.hpp"
#include "logging/log.hpp"
@@ -117,8 +118,8 @@ void ZRelocationSetSelectorGroup::select_inner() {
const int npages = _live_pages.length();
int selected_from = 0;
int selected_to = 0;
- size_t npages_selected[ZPageAgeMax + 1] = { 0 };
- size_t selected_live_bytes[ZPageAgeMax + 1] = { 0 };
+ size_t npages_selected[ZPageAgeCount] = { 0 };
+ size_t selected_live_bytes[ZPageAgeCount] = { 0 };
size_t selected_forwarding_entries = 0;
size_t from_live_bytes = 0;
@@ -149,8 +150,8 @@ void ZRelocationSetSelectorGroup::select_inner() {
if (diff_reclaimable > _fragmentation_limit) {
selected_from = from;
selected_to = to;
- selected_live_bytes[static_cast(page->age())] += page_live_bytes;
- npages_selected[static_cast(page->age())] += 1;
+ selected_live_bytes[untype(page->age())] += page_live_bytes;
+ npages_selected[untype(page->age())] += 1;
selected_forwarding_entries = from_forwarding_entries;
}
@@ -172,7 +173,7 @@ void ZRelocationSetSelectorGroup::select_inner() {
_forwarding_entries = selected_forwarding_entries;
// Update statistics
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
+ for (uint i = 0; i < ZPageAgeCount; ++i) {
_stats[i]._relocate = selected_live_bytes[i];
_stats[i]._npages_selected = npages_selected[i];
}
@@ -200,7 +201,7 @@ void ZRelocationSetSelectorGroup::select() {
}
ZRelocationSetSelectorGroupStats s{};
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
+ for (uint i = 0; i < ZPageAgeCount; ++i) {
s._npages_candidates += _stats[i].npages_candidates();
s._total += _stats[i].total();
s._empty += _stats[i].empty();
@@ -239,8 +240,8 @@ void ZRelocationSetSelector::select() {
ZRelocationSetSelectorStats ZRelocationSetSelector::stats() const {
ZRelocationSetSelectorStats stats;
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- const ZPageAge age = static_cast(i);
+ for (ZPageAge age : ZPageAgeRange()) {
+ const uint i = untype(age);
stats._small[i] = _small.stats(age);
stats._medium[i] = _medium.stats(age);
stats._large[i] = _large.stats(age);
diff --git a/src/hotspot/share/gc/z/zRelocationSetSelector.hpp b/src/hotspot/share/gc/z/zRelocationSetSelector.hpp
index b6ec2347153..21772405f25 100644
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.hpp
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -62,9 +62,9 @@ class ZRelocationSetSelectorStats {
friend class ZRelocationSetSelector;
private:
- ZRelocationSetSelectorGroupStats _small[ZPageAgeMax + 1];
- ZRelocationSetSelectorGroupStats _medium[ZPageAgeMax + 1];
- ZRelocationSetSelectorGroupStats _large[ZPageAgeMax + 1];
+ ZRelocationSetSelectorGroupStats _small[ZPageAgeCount];
+ ZRelocationSetSelectorGroupStats _medium[ZPageAgeCount];
+ ZRelocationSetSelectorGroupStats _large[ZPageAgeCount];
size_t _has_relocatable_pages;
@@ -90,7 +90,7 @@ private:
ZArray _live_pages;
ZArray _not_selected_pages;
size_t _forwarding_entries;
- ZRelocationSetSelectorGroupStats _stats[ZPageAgeMax + 1];
+ ZRelocationSetSelectorGroupStats _stats[ZPageAgeCount];
bool is_disabled();
bool is_selectable();
diff --git a/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp b/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp
index 7cd0a005aac..7740e0764a6 100644
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +29,7 @@
#include "gc/z/zArray.inline.hpp"
#include "gc/z/zGlobals.hpp"
#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageAge.inline.hpp"
#include "utilities/powerOfTwo.hpp"
inline size_t ZRelocationSetSelectorGroupStats::npages_candidates() const {
@@ -60,15 +61,15 @@ inline bool ZRelocationSetSelectorStats::has_relocatable_pages() const {
}
inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::small(ZPageAge age) const {
- return _small[static_cast(age)];
+ return _small[untype(age)];
}
inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::medium(ZPageAge age) const {
- return _medium[static_cast(age)];
+ return _medium[untype(age)];
}
inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::large(ZPageAge age) const {
- return _large[static_cast(age)];
+ return _large[untype(age)];
}
inline bool ZRelocationSetSelectorGroup::pre_filter_page(const ZPage* page, size_t live_bytes) const {
@@ -113,7 +114,7 @@ inline void ZRelocationSetSelectorGroup::register_live_page(ZPage* page) {
}
const size_t size = page->size();
- const uint age = static_cast(page->age());
+ const uint age = untype(page->age());
_stats[age]._npages_candidates++;
_stats[age]._total += size;
_stats[age]._live += live;
@@ -122,7 +123,7 @@ inline void ZRelocationSetSelectorGroup::register_live_page(ZPage* page) {
inline void ZRelocationSetSelectorGroup::register_empty_page(ZPage* page) {
const size_t size = page->size();
- const uint age = static_cast(page->age());
+ const uint age = untype(page->age());
_stats[age]._npages_candidates++;
_stats[age]._total += size;
_stats[age]._empty += size;
@@ -141,7 +142,7 @@ inline size_t ZRelocationSetSelectorGroup::forwarding_entries() const {
}
inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorGroup::stats(ZPageAge age) const {
- return _stats[static_cast(age)];
+ return _stats[untype(age)];
}
inline void ZRelocationSetSelector::register_live_page(ZPage* page) {
@@ -188,8 +189,7 @@ inline void ZRelocationSetSelector::clear_empty_pages() {
inline size_t ZRelocationSetSelector::total() const {
size_t sum = 0;
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- const ZPageAge age = static_cast(i);
+ for (ZPageAge age : ZPageAgeRange()) {
sum += _small.stats(age).total() + _medium.stats(age).total() + _large.stats(age).total();
}
return sum;
@@ -197,8 +197,7 @@ inline size_t ZRelocationSetSelector::total() const {
inline size_t ZRelocationSetSelector::empty() const {
size_t sum = 0;
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- const ZPageAge age = static_cast(i);
+ for (ZPageAge age : ZPageAgeRange()) {
sum += _small.stats(age).empty() + _medium.stats(age).empty() + _large.stats(age).empty();
}
return sum;
@@ -206,8 +205,7 @@ inline size_t ZRelocationSetSelector::empty() const {
inline size_t ZRelocationSetSelector::relocate() const {
size_t sum = 0;
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- const ZPageAge age = static_cast(i);
+ for (ZPageAge age : ZPageAgeRange()) {
sum += _small.stats(age).relocate() + _medium.stats(age).relocate() + _large.stats(age).relocate();
}
return sum;
diff --git a/src/hotspot/share/gc/z/zRemembered.cpp b/src/hotspot/share/gc/z/zRemembered.cpp
index b94d676242c..815b914dd8d 100644
--- a/src/hotspot/share/gc/z/zRemembered.cpp
+++ b/src/hotspot/share/gc/z/zRemembered.cpp
@@ -392,69 +392,71 @@ struct ZRemsetTableEntry {
ZForwarding* _forwarding;
};
-class ZRemsetTableIterator {
-private:
- ZRemembered* const _remembered;
- ZPageTable* const _page_table;
- const ZForwardingTable* const _old_forwarding_table;
- volatile BitMap::idx_t _claimed;
-
-public:
- ZRemsetTableIterator(ZRemembered* remembered)
- : _remembered(remembered),
- _page_table(remembered->_page_table),
- _old_forwarding_table(remembered->_old_forwarding_table),
- _claimed(0) {}
+ZRemsetTableIterator::ZRemsetTableIterator(ZRemembered* remembered, bool previous)
+ : _remembered(remembered),
+ _bm(previous
+ ? _remembered->_found_old.previous_bitmap()
+ : _remembered->_found_old.current_bitmap()),
+ _page_table(remembered->_page_table),
+ _old_forwarding_table(remembered->_old_forwarding_table),
+ _claimed(0) {}
// This iterator uses the "found old" optimization.
- bool next(ZRemsetTableEntry* entry_addr) {
- BitMap* const bm = _remembered->_found_old.previous_bitmap();
+bool ZRemsetTableIterator::next(ZRemsetTableEntry* entry_addr) {
+ BitMap::idx_t prev = Atomic::load(&_claimed);
- BitMap::idx_t prev = Atomic::load(&_claimed);
-
- for (;;) {
- if (prev == bm->size()) {
- return false;
- }
-
- const BitMap::idx_t page_index = bm->find_first_set_bit(_claimed);
- if (page_index == bm->size()) {
- Atomic::cmpxchg(&_claimed, prev, page_index, memory_order_relaxed);
- return false;
- }
-
- const BitMap::idx_t res = Atomic::cmpxchg(&_claimed, prev, page_index + 1, memory_order_relaxed);
- if (res != prev) {
- // Someone else claimed
- prev = res;
- continue;
- }
-
- // Found bit - look around for page or forwarding to scan
-
- ZForwarding* forwarding = nullptr;
- if (ZGeneration::old()->is_phase_relocate()) {
- forwarding = _old_forwarding_table->at(page_index);
- }
-
- ZPage* page = _page_table->at(page_index);
- if (page != nullptr && !page->is_old()) {
- page = nullptr;
- }
-
- if (page == nullptr && forwarding == nullptr) {
- // Nothing to scan
- continue;
- }
-
- // Found old page or old forwarding
- entry_addr->_forwarding = forwarding;
- entry_addr->_page = page;
-
- return true;
+ for (;;) {
+ if (prev == _bm->size()) {
+ return false;
}
+
+ const BitMap::idx_t page_index = _bm->find_first_set_bit(_claimed);
+ if (page_index == _bm->size()) {
+ Atomic::cmpxchg(&_claimed, prev, page_index, memory_order_relaxed);
+ return false;
+ }
+
+ const BitMap::idx_t res = Atomic::cmpxchg(&_claimed, prev, page_index + 1, memory_order_relaxed);
+ if (res != prev) {
+ // Someone else claimed
+ prev = res;
+ continue;
+ }
+
+ // Found bit - look around for page or forwarding to scan
+
+ ZForwarding* forwarding = nullptr;
+ if (ZGeneration::old()->is_phase_relocate()) {
+ forwarding = _old_forwarding_table->at(page_index);
+ }
+
+ ZPage* page = _page_table->at(page_index);
+ if (page != nullptr && !page->is_old()) {
+ page = nullptr;
+ }
+
+ if (page == nullptr && forwarding == nullptr) {
+ // Nothing to scan
+ continue;
+ }
+
+ // Found old page or old forwarding
+ entry_addr->_forwarding = forwarding;
+ entry_addr->_page = page;
+
+ return true;
}
-};
+}
+
+void ZRemembered::remap_current(ZRemsetTableIterator* iter) {
+ for (ZRemsetTableEntry entry; iter->next(&entry);) {
+ assert(entry._forwarding == nullptr, "Shouldn't be looking for forwardings");
+ assert(entry._page != nullptr, "Must have found a page");
+ assert(entry._page->is_old(), "Should only have found old pages");
+
+ entry._page->oops_do_current_remembered(ZBarrier::load_barrier_on_oop_field);
+ }
+}
// This task scans the remembered set and follows pointers when possible.
// Interleaving remembered set scanning with marking makes the marking times
@@ -470,7 +472,7 @@ public:
: ZRestartableTask("ZRememberedScanMarkFollowTask"),
_remembered(remembered),
_mark(mark),
- _remset_table_iterator(remembered) {
+ _remset_table_iterator(remembered, true /* previous */) {
_mark->prepare_work();
_remembered->_page_allocator->enable_safe_destroy();
}
diff --git a/src/hotspot/share/gc/z/zRemembered.hpp b/src/hotspot/share/gc/z/zRemembered.hpp
index 1400472f2bc..6cc76f5dc26 100644
--- a/src/hotspot/share/gc/z/zRemembered.hpp
+++ b/src/hotspot/share/gc/z/zRemembered.hpp
@@ -35,7 +35,9 @@ class ZMark;
class ZPage;
class ZPageAllocator;
class ZPageTable;
+class ZRemsetTableIterator;
struct ZRememberedSetContaining;
+struct ZRemsetTableEntry;
class ZRemembered {
friend class ZRememberedScanMarkFollowTask;
@@ -99,6 +101,26 @@ public:
// Register pages with the remembered set
void register_found_old(ZPage* page);
+
+ // Remap the current remembered set
+ void remap_current(ZRemsetTableIterator* iter);
+};
+
+// This iterator uses the "found old" optimization to skip having to iterate
+// over the entire page table. Make sure to check where and how the FoundOld
+// data is cycled before using this iterator.
+class ZRemsetTableIterator {
+private:
+ ZRemembered* const _remembered;
+ BitMap* const _bm;
+ ZPageTable* const _page_table;
+ const ZForwardingTable* const _old_forwarding_table;
+ volatile BitMap::idx_t _claimed;
+
+public:
+ ZRemsetTableIterator(ZRemembered* remembered, bool previous);
+
+ bool next(ZRemsetTableEntry* entry_addr);
};
#endif // SHARE_GC_Z_ZREMEMBERED_HPP
diff --git a/src/hotspot/share/gc/z/zStat.cpp b/src/hotspot/share/gc/z/zStat.cpp
index ec751cbb693..bfde904c1e7 100644
--- a/src/hotspot/share/gc/z/zStat.cpp
+++ b/src/hotspot/share/gc/z/zStat.cpp
@@ -30,6 +30,7 @@
#include "gc/z/zGeneration.inline.hpp"
#include "gc/z/zGlobals.hpp"
#include "gc/z/zNMethodTable.hpp"
+#include "gc/z/zPageAge.inline.hpp"
#include "gc/z/zPageAllocator.inline.hpp"
#include "gc/z/zRelocationSetSelector.inline.hpp"
#include "gc/z/zStat.hpp"
@@ -1499,9 +1500,7 @@ void ZStatRelocation::print_page_summary() {
summary.relocate += stats.relocate();
};
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- const ZPageAge age = static_cast(i);
-
+ for (ZPageAge age : ZPageAgeRange()) {
account_page_size(small_summary, _selector_stats.small(age));
account_page_size(medium_summary, _selector_stats.medium(age));
account_page_size(large_summary, _selector_stats.large(age));
@@ -1557,13 +1556,13 @@ void ZStatRelocation::print_age_table() {
.center("Large")
.end());
- size_t live[ZPageAgeMax + 1] = {};
- size_t total[ZPageAgeMax + 1] = {};
+ size_t live[ZPageAgeCount] = {};
+ size_t total[ZPageAgeCount] = {};
uint oldest_none_empty_age = 0;
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- ZPageAge age = static_cast(i);
+ for (ZPageAge age : ZPageAgeRange()) {
+ uint i = untype(age);
auto summarize_pages = [&](const ZRelocationSetSelectorGroupStats& stats) {
live[i] += stats.live();
total[i] += stats.total();
@@ -1579,7 +1578,7 @@ void ZStatRelocation::print_age_table() {
}
for (uint i = 0; i <= oldest_none_empty_age; ++i) {
- ZPageAge age = static_cast(i);
+ ZPageAge age = to_zpageage(i);
FormatBuffer<> age_str("");
if (age == ZPageAge::eden) {
@@ -1791,8 +1790,7 @@ void ZStatHeap::at_select_relocation_set(const ZRelocationSetSelectorStats& stat
ZLocker locker(&_stat_lock);
size_t live = 0;
- for (uint i = 0; i <= ZPageAgeMax; ++i) {
- const ZPageAge age = static_cast(i);
+ for (ZPageAge age : ZPageAgeRange()) {
live += stats.small(age).live() + stats.medium(age).live() + stats.large(age).live();
}
_at_mark_end.live = live;
diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp
index e1f525e5372..d818cf9fc3f 100644
--- a/src/hotspot/share/gc/z/z_globals.hpp
+++ b/src/hotspot/share/gc/z/z_globals.hpp
@@ -113,7 +113,7 @@
\
product(int, ZTenuringThreshold, -1, DIAGNOSTIC, \
"Young generation tenuring threshold, -1 for dynamic computation")\
- range(-1, static_cast(ZPageAgeMax)) \
+ range(-1, static_cast(ZPageAgeCount) - 1) \
\
develop(bool, ZVerifyOops, false, \
"Verify accessed oops") \
diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h
index 61857ac3ccd..73f60765a70 100644
--- a/src/hotspot/share/include/jvm.h
+++ b/src/hotspot/share/include/jvm.h
@@ -301,6 +301,9 @@ JVM_HoldsLock(JNIEnv *env, jclass threadClass, jobject obj);
JNIEXPORT jobject JNICALL
JVM_GetStackTrace(JNIEnv *env, jobject thread);
+JNIEXPORT jobject JNICALL
+JVM_CreateThreadSnapshot(JNIEnv* env, jobject thread);
+
JNIEXPORT jobjectArray JNICALL
JVM_GetAllThreads(JNIEnv *env, jclass dummy);
diff --git a/src/hotspot/share/interpreter/linkResolver.cpp b/src/hotspot/share/interpreter/linkResolver.cpp
index b26c643d15f..22199baef8e 100644
--- a/src/hotspot/share/interpreter/linkResolver.cpp
+++ b/src/hotspot/share/interpreter/linkResolver.cpp
@@ -1200,7 +1200,7 @@ Method* LinkResolver::linktime_resolve_special_method(const LinkInfo& link_info,
}
// ensure that invokespecial's interface method reference is in
- // a direct superinterface, not an indirect superinterface
+ // a direct superinterface, not an indirect superinterface or unrelated interface
Klass* current_klass = link_info.current_klass();
if (current_klass != nullptr && resolved_klass->is_interface()) {
InstanceKlass* klass_to_check = InstanceKlass::cast(current_klass);
@@ -1209,7 +1209,7 @@ Method* LinkResolver::linktime_resolve_special_method(const LinkInfo& link_info,
stringStream ss;
ss.print("Interface method reference: '");
resolved_method->print_external_name(&ss);
- ss.print("', is in an indirect superinterface of %s",
+ ss.print("', is not in a direct superinterface of %s",
current_klass->external_name());
THROW_MSG_NULL(vmSymbols::java_lang_IncompatibleClassChangeError(), ss.as_string());
}
diff --git a/src/hotspot/share/jfr/jfr.inline.hpp b/src/hotspot/share/jfr/jfr.inline.hpp
index bdb47f600e6..5b6fc62d0ea 100644
--- a/src/hotspot/share/jfr/jfr.inline.hpp
+++ b/src/hotspot/share/jfr/jfr.inline.hpp
@@ -32,7 +32,8 @@
inline bool Jfr::has_sample_request(JavaThread* jt) {
assert(jt != nullptr, "invariant");
- return jt->jfr_thread_local()->has_sample_request();
+ JfrThreadLocal* tl = jt->jfr_thread_local();
+ return tl->has_sample_request() || tl->has_cpu_time_jfr_requests();
}
inline void Jfr::check_and_process_sample_request(JavaThread* jt) {
diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp
index 6f1c1936574..bc2412a90c1 100644
--- a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp
+++ b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp
@@ -24,6 +24,7 @@
#include "jfr/jfr.hpp"
#include "jfr/jfrEvents.hpp"
+#include "jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp"
#include "jfr/periodic/sampling/jfrThreadSampler.hpp"
#include "jfr/recorder/jfrEventSetting.hpp"
#include "jfr/recorder/jfrRecorder.hpp"
@@ -169,6 +170,11 @@ NO_TRANSITION(jboolean, jfr_set_throttle(JNIEnv* env, jclass jvm, jlong event_ty
return JNI_TRUE;
NO_TRANSITION_END
+JVM_ENTRY_NO_ENV(void, jfr_set_cpu_throttle(JNIEnv* env, jclass jvm, jdouble rate, jboolean auto_adapt))
+ JfrEventSetting::set_enabled(JfrCPUTimeSampleEvent, rate > 0);
+ JfrCPUTimeThreadSampling::set_rate(rate, auto_adapt == JNI_TRUE);
+JVM_END
+
NO_TRANSITION(void, jfr_set_miscellaneous(JNIEnv* env, jclass jvm, jlong event_type_id, jlong value))
JfrEventSetting::set_miscellaneous(event_type_id, value);
const JfrEventId typed_event_id = (JfrEventId)event_type_id;
diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp
index 9c78c6239d4..dbea7f0180d 100644
--- a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp
+++ b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp
@@ -129,6 +129,8 @@ jlong JNICALL jfr_get_unloaded_event_classes_count(JNIEnv* env, jclass jvm);
jboolean JNICALL jfr_set_throttle(JNIEnv* env, jclass jvm, jlong event_type_id, jlong event_sample_size, jlong period_ms);
+void JNICALL jfr_set_cpu_throttle(JNIEnv* env, jclass jvm, jdouble rate, jboolean auto_adapt);
+
void JNICALL jfr_set_miscellaneous(JNIEnv* env, jclass jvm, jlong id, jlong value);
void JNICALL jfr_emit_old_object_samples(JNIEnv* env, jclass jvm, jlong cutoff_ticks, jboolean, jboolean);
diff --git a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp
index 33a564dee2f..82ef93d95b2 100644
--- a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp
+++ b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp
@@ -83,6 +83,7 @@ JfrJniMethodRegistration::JfrJniMethodRegistration(JNIEnv* env) {
(char*)"getUnloadedEventClassCount", (char*)"()J", (void*)jfr_get_unloaded_event_classes_count,
(char*)"setMiscellaneous", (char*)"(JJ)V", (void*)jfr_set_miscellaneous,
(char*)"setThrottle", (char*)"(JJJ)Z", (void*)jfr_set_throttle,
+ (char*)"setCPUThrottle", (char*)"(DZ)V", (void*)jfr_set_cpu_throttle,
(char*)"emitOldObjectSamples", (char*)"(JZZ)V", (void*)jfr_emit_old_object_samples,
(char*)"shouldRotateDisk", (char*)"()Z", (void*)jfr_should_rotate_disk,
(char*)"exclude", (char*)"(Ljava/lang/Thread;)V", (void*)jfr_exclude_thread,
diff --git a/src/hotspot/share/jfr/metadata/metadata.xml b/src/hotspot/share/jfr/metadata/metadata.xml
index 9c04ec3dca1..d7287adb1a6 100644
--- a/src/hotspot/share/jfr/metadata/metadata.xml
+++ b/src/hotspot/share/jfr/metadata/metadata.xml
@@ -962,6 +962,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1299,7 +1315,9 @@
-
+
+
+
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrCPUTimeThreadSampler.cpp b/src/hotspot/share/jfr/periodic/sampling/jfrCPUTimeThreadSampler.cpp
new file mode 100644
index 00000000000..2f063123a3d
--- /dev/null
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrCPUTimeThreadSampler.cpp
@@ -0,0 +1,780 @@
+/*
+ * Copyright (c) 2025 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp"
+#include "logging/log.hpp"
+
+
+#if defined(LINUX)
+#include "jfr/periodic/sampling/jfrThreadSampling.hpp"
+#include "jfr/support/jfrThreadLocal.hpp"
+#include "jfr/utilities/jfrTime.hpp"
+#include "jfr/utilities/jfrThreadIterator.hpp"
+#include "jfr/utilities/jfrTypes.hpp"
+#include "jfrfiles/jfrEventClasses.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/osThread.hpp"
+#include "runtime/safepointMechanism.inline.hpp"
+#include "runtime/threadSMR.hpp"
+#include "runtime/vmOperation.hpp"
+#include "runtime/vmThread.hpp"
+#include "utilities/ticks.hpp"
+
+#include "signals_posix.hpp"
+
+static const int64_t AUTOADAPT_INTERVAL_MS = 100;
+
+static bool is_excluded(JavaThread* jt) {
+ return jt->is_hidden_from_external_view() ||
+ jt->jfr_thread_local()->is_excluded() ||
+ jt->is_JfrRecorder_thread();
+}
+
+static JavaThread* get_java_thread_if_valid() {
+ Thread* raw_thread = Thread::current_or_null_safe();
+ if (raw_thread == nullptr) {
+ // probably while shutting down
+ return nullptr;
+ }
+ assert(raw_thread->is_Java_thread(), "invariant");
+ JavaThread* jt = JavaThread::cast(raw_thread);
+ if (is_excluded(jt) || jt->is_exiting()) {
+ return nullptr;
+ }
+ return jt;
+}
+
+JfrCPUTimeTraceQueue::JfrCPUTimeTraceQueue(u4 capacity) :
+ _data(nullptr), _capacity(capacity), _head(0), _lost_samples(0) {
+ if (capacity != 0) {
+ _data = JfrCHeapObj::new_array(capacity);
+ }
+}
+
+JfrCPUTimeTraceQueue::~JfrCPUTimeTraceQueue() {
+ if (_data != nullptr) {
+ assert(_capacity != 0, "invariant");
+ JfrCHeapObj::free(_data, _capacity * sizeof(JfrCPUTimeSampleRequest));
+ }
+}
+
+bool JfrCPUTimeTraceQueue::enqueue(JfrCPUTimeSampleRequest& request) {
+ assert(JavaThread::current()->jfr_thread_local()->is_cpu_time_jfr_enqueue_locked(), "invariant");
+ assert(&JavaThread::current()->jfr_thread_local()->cpu_time_jfr_queue() == this, "invariant");
+ u4 elementIndex;
+ do {
+ elementIndex = Atomic::load_acquire(&_head);
+ if (elementIndex >= _capacity) {
+ return false;
+ }
+ } while (Atomic::cmpxchg(&_head, elementIndex, elementIndex + 1) != elementIndex);
+ _data[elementIndex] = request;
+ return true;
+}
+
+JfrCPUTimeSampleRequest& JfrCPUTimeTraceQueue::at(u4 index) {
+ assert(index < _head, "invariant");
+ return _data[index];
+}
+
+static volatile u4 _lost_samples_sum = 0;
+
+u4 JfrCPUTimeTraceQueue::size() const {
+ return Atomic::load_acquire(&_head);
+}
+
+void JfrCPUTimeTraceQueue::set_size(u4 size) {
+ Atomic::release_store(&_head, size);
+}
+
+u4 JfrCPUTimeTraceQueue::capacity() const {
+ return _capacity;
+}
+
+void JfrCPUTimeTraceQueue::set_capacity(u4 capacity) {
+ _head = 0;
+ if (_data != nullptr) {
+ assert(_capacity != 0, "invariant");
+ JfrCHeapObj::free(_data, _capacity * sizeof(JfrCPUTimeSampleRequest));
+ }
+ if (capacity != 0) {
+ _data = JfrCHeapObj::new_array(capacity);
+ } else {
+ _data = nullptr;
+ }
+ _capacity = capacity;
+}
+
+bool JfrCPUTimeTraceQueue::is_empty() const {
+ return Atomic::load_acquire(&_head) == 0;
+}
+
+u4 JfrCPUTimeTraceQueue::lost_samples() const {
+ return Atomic::load(&_lost_samples);
+}
+
+void JfrCPUTimeTraceQueue::increment_lost_samples() {
+ Atomic::inc(&_lost_samples_sum);
+ Atomic::inc(&_lost_samples);
+}
+
+u4 JfrCPUTimeTraceQueue::get_and_reset_lost_samples() {
+ return Atomic::xchg(&_lost_samples, (u4)0);
+}
+
+void JfrCPUTimeTraceQueue::resize(u4 capacity) {
+ if (capacity != _capacity) {
+ set_capacity(capacity);
+ }
+}
+
+void JfrCPUTimeTraceQueue::resize_for_period(u4 period_millis) {
+ u4 capacity = CPU_TIME_QUEUE_CAPACITY;
+ if (period_millis > 0 && period_millis < 10) {
+ capacity = (u4) ((double) capacity * 10 / period_millis);
+ }
+ resize(capacity);
+}
+
+void JfrCPUTimeTraceQueue::clear() {
+ Atomic::release_store(&_head, (u4)0);
+}
+
+static int64_t compute_sampling_period(double rate) {
+ if (rate == 0) {
+ return 0;
+ }
+ return os::active_processor_count() * 1000000000.0 / rate;
+}
+
+class JfrCPUSamplerThread : public NonJavaThread {
+ friend class JfrCPUTimeThreadSampling;
+ private:
+ Semaphore _sample;
+ NonJavaThread* _sampler_thread;
+ double _rate;
+ bool _auto_adapt;
+ volatile int64_t _current_sampling_period_ns;
+ volatile bool _disenrolled;
+ // top bit is used to indicate that no signal handler should proceed
+ volatile u4 _active_signal_handlers;
+ volatile bool _is_async_processing_of_cpu_time_jfr_requests_triggered;
+ volatile bool _warned_about_timer_creation_failure;
+ volatile bool _signal_handler_installed;
+
+ static const u4 STOP_SIGNAL_BIT = 0x80000000;
+
+ JfrCPUSamplerThread(double rate, bool auto_adapt);
+
+ void start_thread();
+
+ void enroll();
+ void disenroll();
+ void update_all_thread_timers();
+
+ void auto_adapt_period_if_needed();
+
+ void set_rate(double rate, bool auto_adapt);
+ int64_t get_sampling_period() const { return Atomic::load(&_current_sampling_period_ns); };
+
+ void sample_thread(JfrSampleRequest& request, void* ucontext, JavaThread* jt, JfrThreadLocal* tl, JfrTicks& now);
+
+ // process the queues for all threads that are in native state (and requested to be processed)
+ void stackwalk_threads_in_native();
+ bool create_timer_for_thread(JavaThread* thread, timer_t &timerid);
+
+ void stop_signal_handlers();
+
+ // returns false if the stop signal bit was set, true otherwise
+ bool increment_signal_handler_count();
+
+ void decrement_signal_handler_count();
+
+ void initialize_active_signal_handler_counter();
+
+protected:
+ virtual void post_run();
+public:
+ virtual const char* name() const { return "JFR CPU Sampler Thread"; }
+ virtual const char* type_name() const { return "JfrCPUTimeSampler"; }
+ void run();
+ void on_javathread_create(JavaThread* thread);
+ void on_javathread_terminate(JavaThread* thread);
+
+ void handle_timer_signal(siginfo_t* info, void* context);
+ bool init_timers();
+ void stop_timer();
+
+ void trigger_async_processing_of_cpu_time_jfr_requests();
+};
+
+JfrCPUSamplerThread::JfrCPUSamplerThread(double rate, bool auto_adapt) :
+ _sample(),
+ _sampler_thread(nullptr),
+ _rate(rate),
+ _auto_adapt(auto_adapt),
+ _current_sampling_period_ns(compute_sampling_period(rate)),
+ _disenrolled(true),
+ _active_signal_handlers(STOP_SIGNAL_BIT),
+ _is_async_processing_of_cpu_time_jfr_requests_triggered(false),
+ _warned_about_timer_creation_failure(false),
+ _signal_handler_installed(false) {
+ assert(rate >= 0, "invariant");
+}
+
+void JfrCPUSamplerThread::trigger_async_processing_of_cpu_time_jfr_requests() {
+ Atomic::release_store(&_is_async_processing_of_cpu_time_jfr_requests_triggered, true);
+}
+
+void JfrCPUSamplerThread::on_javathread_create(JavaThread* thread) {
+ if (thread->is_hidden_from_external_view() || thread->is_JfrRecorder_thread() ||
+ !Atomic::load_acquire(&_signal_handler_installed)) {
+ return;
+ }
+ JfrThreadLocal* tl = thread->jfr_thread_local();
+ assert(tl != nullptr, "invariant");
+ tl->cpu_time_jfr_queue().resize_for_period(_current_sampling_period_ns / 1000000);
+ timer_t timerid;
+ if (create_timer_for_thread(thread, timerid)) {
+ tl->set_cpu_timer(&timerid);
+ } else {
+ if (!Atomic::or_then_fetch(&_warned_about_timer_creation_failure, true)) {
+ log_warning(jfr)("Failed to create timer for a thread");
+ }
+ tl->deallocate_cpu_time_jfr_queue();
+ }
+}
+
+void JfrCPUSamplerThread::on_javathread_terminate(JavaThread* thread) {
+ JfrThreadLocal* tl = thread->jfr_thread_local();
+ assert(tl != nullptr, "invariant");
+ timer_t* timer = tl->cpu_timer();
+ if (timer == nullptr) {
+ return; // no timer was created for this thread
+ }
+ tl->unset_cpu_timer();
+ tl->deallocate_cpu_time_jfr_queue();
+ s4 lost_samples = tl->cpu_time_jfr_queue().lost_samples();
+ if (lost_samples > 0) {
+ JfrCPUTimeThreadSampling::send_lost_event(JfrTicks::now(), JfrThreadLocal::thread_id(thread), lost_samples);
+ }
+}
+
+void JfrCPUSamplerThread::start_thread() {
+ if (os::create_thread(this, os::os_thread)) {
+ os::start_thread(this);
+ } else {
+ log_error(jfr)("Failed to create thread for thread sampling");
+ }
+}
+
+void JfrCPUSamplerThread::enroll() {
+ if (Atomic::cmpxchg(&_disenrolled, true, false)) {
+ Atomic::store(&_warned_about_timer_creation_failure, false);
+ initialize_active_signal_handler_counter();
+ log_trace(jfr)("Enrolling CPU thread sampler");
+ _sample.signal();
+ if (!init_timers()) {
+ log_error(jfr)("Failed to initialize timers for CPU thread sampler");
+ disenroll();
+ return;
+ }
+ log_trace(jfr)("Enrolled CPU thread sampler");
+ }
+}
+
+void JfrCPUSamplerThread::disenroll() {
+ if (!Atomic::cmpxchg(&_disenrolled, false, true)) {
+ log_trace(jfr)("Disenrolling CPU thread sampler");
+ if (Atomic::load_acquire(&_signal_handler_installed)) {
+ stop_timer();
+ stop_signal_handlers();
+ }
+ _sample.wait();
+ log_trace(jfr)("Disenrolled CPU thread sampler");
+ }
+}
+
+void JfrCPUSamplerThread::run() {
+ assert(_sampler_thread == nullptr, "invariant");
+ _sampler_thread = this;
+ int64_t last_auto_adapt_check = os::javaTimeNanos();
+ while (true) {
+ if (!_sample.trywait()) {
+ // disenrolled
+ _sample.wait();
+ }
+ _sample.signal();
+
+ if (os::javaTimeNanos() - last_auto_adapt_check > AUTOADAPT_INTERVAL_MS * 1000000) {
+ auto_adapt_period_if_needed();
+ last_auto_adapt_check = os::javaTimeNanos();
+ }
+
+ if (Atomic::cmpxchg(&_is_async_processing_of_cpu_time_jfr_requests_triggered, true, false)) {
+ stackwalk_threads_in_native();
+ }
+ os::naked_sleep(100);
+ }
+}
+
+void JfrCPUSamplerThread::stackwalk_threads_in_native() {
+ ResourceMark rm;
+ // Required to prevent JFR from sampling through an ongoing safepoint
+ MutexLocker tlock(Threads_lock);
+ ThreadsListHandle tlh;
+ Thread* current = Thread::current();
+ for (size_t i = 0; i < tlh.list()->length(); i++) {
+ JavaThread* jt = tlh.list()->thread_at(i);
+ JfrThreadLocal* tl = jt->jfr_thread_local();
+ if (tl->wants_async_processing_of_cpu_time_jfr_requests()) {
+ if (jt->thread_state() != _thread_in_native || !tl->try_acquire_cpu_time_jfr_dequeue_lock()) {
+ tl->set_do_async_processing_of_cpu_time_jfr_requests(false);
+ continue;
+ }
+ if (jt->has_last_Java_frame()) {
+ JfrThreadSampling::process_cpu_time_request(jt, tl, current, false);
+ } else {
+ tl->set_do_async_processing_of_cpu_time_jfr_requests(false);
+ }
+ tl->release_cpu_time_jfr_queue_lock();
+ }
+ }
+}
+
+static volatile size_t count = 0;
+
+void JfrCPUTimeThreadSampling::send_empty_event(const JfrTicks &start_time, traceid tid, Tickspan cpu_time_period) {
+ EventCPUTimeSample event(UNTIMED);
+ event.set_failed(true);
+ event.set_starttime(start_time);
+ event.set_eventThread(tid);
+ event.set_stackTrace(0);
+ event.set_samplingPeriod(cpu_time_period);
+ event.set_biased(false);
+ event.commit();
+}
+
+
+static volatile size_t biased_count = 0;
+
+void JfrCPUTimeThreadSampling::send_event(const JfrTicks &start_time, traceid sid, traceid tid, Tickspan cpu_time_period, bool biased) {
+ EventCPUTimeSample event(UNTIMED);
+ event.set_failed(false);
+ event.set_starttime(start_time);
+ event.set_eventThread(tid);
+ event.set_stackTrace(sid);
+ event.set_samplingPeriod(cpu_time_period);
+ event.set_biased(biased);
+ event.commit();
+ Atomic::inc(&count);
+ if (biased) {
+ Atomic::inc(&biased_count);
+ }
+ if (Atomic::load(&count) % 1000 == 0) {
+ log_debug(jfr)("CPU thread sampler sent %zu events, lost %d, biased %zu\n", Atomic::load(&count), Atomic::load(&_lost_samples_sum), Atomic::load(&biased_count));
+ }
+}
+
+void JfrCPUTimeThreadSampling::send_lost_event(const JfrTicks &time, traceid tid, s4 lost_samples) {
+ if (!EventCPUTimeSamplesLost::is_enabled()) {
+ return;
+ }
+ EventCPUTimeSamplesLost event(UNTIMED);
+ event.set_starttime(time);
+ event.set_lostSamples(lost_samples);
+ event.set_eventThread(tid);
+ event.commit();
+}
+
+void JfrCPUSamplerThread::post_run() {
+ this->NonJavaThread::post_run();
+ delete this;
+}
+
+static JfrCPUTimeThreadSampling* _instance = nullptr;
+
+JfrCPUTimeThreadSampling& JfrCPUTimeThreadSampling::instance() {
+ return *_instance;
+}
+
+JfrCPUTimeThreadSampling* JfrCPUTimeThreadSampling::create() {
+ assert(_instance == nullptr, "invariant");
+ _instance = new JfrCPUTimeThreadSampling();
+ return _instance;
+}
+
+void JfrCPUTimeThreadSampling::destroy() {
+ if (_instance != nullptr) {
+ delete _instance;
+ _instance = nullptr;
+ }
+}
+
+JfrCPUTimeThreadSampling::JfrCPUTimeThreadSampling() : _sampler(nullptr) {}
+
+JfrCPUTimeThreadSampling::~JfrCPUTimeThreadSampling() {
+ if (_sampler != nullptr) {
+ _sampler->disenroll();
+ }
+}
+
+void JfrCPUTimeThreadSampling::create_sampler(double rate, bool auto_adapt) {
+ assert(_sampler == nullptr, "invariant");
+ _sampler = new JfrCPUSamplerThread(rate, auto_adapt);
+ _sampler->start_thread();
+ _sampler->enroll();
+}
+
+void JfrCPUTimeThreadSampling::update_run_state(double rate, bool auto_adapt) {
+ if (rate != 0) {
+ if (_sampler == nullptr) {
+ create_sampler(rate, auto_adapt);
+ } else {
+ _sampler->set_rate(rate, auto_adapt);
+ _sampler->enroll();
+ }
+ return;
+ }
+ if (_sampler != nullptr) {
+ _sampler->set_rate(rate /* 0 */, auto_adapt);
+ _sampler->disenroll();
+ }
+}
+
+void JfrCPUTimeThreadSampling::set_rate(double rate, bool auto_adapt) {
+ assert(rate >= 0, "invariant");
+ if (_instance == nullptr) {
+ return;
+ }
+ instance().set_rate_value(rate, auto_adapt);
+}
+
+void JfrCPUTimeThreadSampling::set_rate_value(double rate, bool auto_adapt) {
+ if (_sampler != nullptr) {
+ _sampler->set_rate(rate, auto_adapt);
+ }
+ update_run_state(rate, auto_adapt);
+}
+
+void JfrCPUTimeThreadSampling::on_javathread_create(JavaThread *thread) {
+ if (_instance != nullptr && _instance->_sampler != nullptr) {
+ _instance->_sampler->on_javathread_create(thread);
+ }
+}
+
+void JfrCPUTimeThreadSampling::on_javathread_terminate(JavaThread *thread) {
+ if (_instance != nullptr && _instance->_sampler != nullptr) {
+ _instance->_sampler->on_javathread_terminate(thread);
+ }
+}
+
+void JfrCPUTimeThreadSampling::trigger_async_processing_of_cpu_time_jfr_requests() {
+ if (_instance != nullptr && _instance->_sampler != nullptr) {
+ _instance->_sampler->trigger_async_processing_of_cpu_time_jfr_requests();
+ }
+}
+
+void handle_timer_signal(int signo, siginfo_t* info, void* context) {
+ assert(_instance != nullptr, "invariant");
+ _instance->handle_timer_signal(info, context);
+}
+
+
+void JfrCPUTimeThreadSampling::handle_timer_signal(siginfo_t* info, void* context) {
+ if (info->si_code != SI_TIMER) {
+ // not the signal we are interested in
+ return;
+ }
+ assert(_sampler != nullptr, "invariant");
+
+ if (!_sampler->increment_signal_handler_count()) {
+ return;
+ }
+ _sampler->handle_timer_signal(info, context);
+ _sampler->decrement_signal_handler_count();
+}
+
+void JfrCPUSamplerThread::sample_thread(JfrSampleRequest& request, void* ucontext, JavaThread* jt, JfrThreadLocal* tl, JfrTicks& now) {
+ JfrSampleRequestBuilder::build_cpu_time_sample_request(request, ucontext, jt, jt->jfr_thread_local(), now);
+}
+
+static bool check_state(JavaThread* thread) {
+ switch (thread->thread_state()) {
+ case _thread_in_Java:
+ case _thread_in_native:
+ return true;
+ default:
+ return false;
+ }
+}
+
+void JfrCPUSamplerThread::handle_timer_signal(siginfo_t* info, void* context) {
+ JfrTicks now = JfrTicks::now();
+ JavaThread* jt = get_java_thread_if_valid();
+ if (jt == nullptr) {
+ return;
+ }
+ JfrThreadLocal* tl = jt->jfr_thread_local();
+ JfrCPUTimeTraceQueue& queue = tl->cpu_time_jfr_queue();
+ if (!check_state(jt)) {
+ queue.increment_lost_samples();
+ return;
+ }
+ if (!tl->try_acquire_cpu_time_jfr_enqueue_lock()) {
+ queue.increment_lost_samples();
+ return;
+ }
+
+ JfrCPUTimeSampleRequest request;
+ // the sampling period might be too low for the current Linux configuration
+ // so samples might be skipped and we have to compute the actual period
+ int64_t period = get_sampling_period() * (info->si_overrun + 1);
+ request._cpu_time_period = Ticks(period / 1000000000.0 * JfrTime::frequency()) - Ticks(0);
+ sample_thread(request._request, context, jt, tl, now);
+
+ if (queue.enqueue(request)) {
+ if (queue.size() == 1) {
+ tl->set_has_cpu_time_jfr_requests(true);
+ SafepointMechanism::arm_local_poll_release(jt);
+ }
+ } else {
+ queue.increment_lost_samples();
+ }
+
+ if (jt->thread_state() == _thread_in_native) {
+ if (!tl->wants_async_processing_of_cpu_time_jfr_requests()) {
+ tl->set_do_async_processing_of_cpu_time_jfr_requests(true);
+ JfrCPUTimeThreadSampling::trigger_async_processing_of_cpu_time_jfr_requests();
+ }
+ } else {
+ tl->set_do_async_processing_of_cpu_time_jfr_requests(false);
+ }
+
+ tl->release_cpu_time_jfr_queue_lock();
+}
+
+static const int SIG = SIGPROF;
+
+static void set_timer_time(timer_t timerid, int64_t period_nanos) {
+ struct itimerspec its;
+ if (period_nanos == 0) {
+ its.it_interval.tv_sec = 0;
+ its.it_interval.tv_nsec = 0;
+ } else {
+ its.it_interval.tv_sec = period_nanos / NANOSECS_PER_SEC;
+ its.it_interval.tv_nsec = period_nanos % NANOSECS_PER_SEC;
+ }
+ its.it_value = its.it_interval;
+ if (timer_settime(timerid, 0, &its, nullptr) == -1) {
+ warning("Failed to set timer for thread sampling: %s", os::strerror(os::get_last_error()));
+ }
+}
+
+bool JfrCPUSamplerThread::create_timer_for_thread(JavaThread* thread, timer_t& timerid) {
+ struct sigevent sev;
+ sev.sigev_notify = SIGEV_THREAD_ID;
+ sev.sigev_signo = SIG;
+ sev.sigev_value.sival_ptr = nullptr;
+ ((int*)&sev.sigev_notify)[1] = thread->osthread()->thread_id();
+ clockid_t clock;
+ int err = pthread_getcpuclockid(thread->osthread()->pthread_id(), &clock);
+ if (err != 0) {
+ log_error(jfr)("Failed to get clock for thread sampling: %s", os::strerror(err));
+ return false;
+ }
+ if (timer_create(clock, &sev, &timerid) < 0) {
+ return false;
+ }
+ int64_t period = get_sampling_period();
+ if (period != 0) {
+ set_timer_time(timerid, period);
+ }
+ return true;
+}
+
+
+void JfrCPUSamplerThread::stop_signal_handlers() {
+ // set the stop signal bit
+ Atomic::or_then_fetch(&_active_signal_handlers, STOP_SIGNAL_BIT, memory_order_acq_rel);
+ while (Atomic::load_acquire(&_active_signal_handlers) > STOP_SIGNAL_BIT) {
+ // wait for all signal handlers to finish
+ os::naked_short_nanosleep(1000);
+ }
+}
+
+// returns false if the stop signal bit was set, true otherwise
+bool JfrCPUSamplerThread::increment_signal_handler_count() {
+ // increment the count of active signal handlers
+ u4 old_value = Atomic::fetch_then_add(&_active_signal_handlers, (u4)1, memory_order_acq_rel);
+ if ((old_value & STOP_SIGNAL_BIT) != 0) {
+ // if the stop signal bit was set, we are not allowed to increment
+ Atomic::dec(&_active_signal_handlers, memory_order_acq_rel);
+ return false;
+ }
+ return true;
+}
+
+void JfrCPUSamplerThread::decrement_signal_handler_count() {
+ Atomic::dec(&_active_signal_handlers, memory_order_acq_rel);
+}
+
+void JfrCPUSamplerThread::initialize_active_signal_handler_counter() {
+ Atomic::release_store(&_active_signal_handlers, (u4)0);
+}
+
+class VM_JFRInitializeCPUTimeSampler : public VM_Operation {
+ private:
+ JfrCPUSamplerThread* const _sampler;
+
+ public:
+ VM_JFRInitializeCPUTimeSampler(JfrCPUSamplerThread* sampler) : _sampler(sampler) {}
+
+ VMOp_Type type() const { return VMOp_JFRInitializeCPUTimeSampler; }
+ void doit() {
+ JfrJavaThreadIterator iter;
+ while (iter.has_next()) {
+ _sampler->on_javathread_create(iter.next());
+ }
+ };
+};
+
+bool JfrCPUSamplerThread::init_timers() {
+ // install sig handler for sig
+ void* prev_handler = PosixSignals::get_signal_handler_for_signal(SIG);
+ if ((prev_handler != SIG_DFL && prev_handler != SIG_IGN && prev_handler != (void*)::handle_timer_signal) ||
+ PosixSignals::install_generic_signal_handler(SIG, (void*)::handle_timer_signal) == (void*)-1) {
+ log_error(jfr)("Conflicting SIGPROF handler found: %p. CPUTimeSample events will not be recorded", prev_handler);
+ return false;
+ }
+ Atomic::release_store(&_signal_handler_installed, true);
+ VM_JFRInitializeCPUTimeSampler op(this);
+ VMThread::execute(&op);
+ return true;
+}
+
+class VM_JFRTerminateCPUTimeSampler : public VM_Operation {
+ private:
+ JfrCPUSamplerThread* const _sampler;
+
+ public:
+ VM_JFRTerminateCPUTimeSampler(JfrCPUSamplerThread* sampler) : _sampler(sampler) {}
+
+ VMOp_Type type() const { return VMOp_JFRTerminateCPUTimeSampler; }
+ void doit() {
+ JfrJavaThreadIterator iter;
+ while (iter.has_next()) {
+ JavaThread *thread = iter.next();
+ JfrThreadLocal* tl = thread->jfr_thread_local();
+ timer_t* timer = tl->cpu_timer();
+ if (timer == nullptr) {
+ continue;
+ }
+ tl->deallocate_cpu_time_jfr_queue();
+ tl->unset_cpu_timer();
+ }
+ };
+};
+
+void JfrCPUSamplerThread::stop_timer() {
+ VM_JFRTerminateCPUTimeSampler op(this);
+ VMThread::execute(&op);
+}
+
+void JfrCPUSamplerThread::auto_adapt_period_if_needed() {
+ int64_t current_period = get_sampling_period();
+ if (_auto_adapt || current_period == -1) {
+ int64_t period = compute_sampling_period(_rate);
+ if (period != current_period) {
+ Atomic::store(&_current_sampling_period_ns, period);
+ update_all_thread_timers();
+ }
+ }
+}
+
+void JfrCPUSamplerThread::set_rate(double rate, bool auto_adapt) {
+ _rate = rate;
+ _auto_adapt = auto_adapt;
+ if (_rate > 0 && Atomic::load_acquire(&_disenrolled) == false) {
+ auto_adapt_period_if_needed();
+ } else {
+ Atomic::store(&_current_sampling_period_ns, compute_sampling_period(rate));
+ }
+}
+
+void JfrCPUSamplerThread::update_all_thread_timers() {
+ int64_t period_millis = get_sampling_period();
+ ThreadsListHandle tlh;
+ for (size_t i = 0; i < tlh.length(); i++) {
+ JavaThread* thread = tlh.thread_at(i);
+ JfrThreadLocal* tl = thread->jfr_thread_local();
+ assert(tl != nullptr, "invariant");
+ timer_t* timer = tl->cpu_timer();
+ if (timer != nullptr) {
+ set_timer_time(*timer, period_millis);
+ }
+ }
+}
+
+#else
+
+static void warn() {
+ static bool displayed_warning = false;
+ if (!displayed_warning) {
+ warning("CPU time method sampling not supported in JFR on your platform");
+ displayed_warning = true;
+ }
+}
+
+static JfrCPUTimeThreadSampling* _instance = nullptr;
+
+JfrCPUTimeThreadSampling& JfrCPUTimeThreadSampling::instance() {
+ return *_instance;
+}
+
+JfrCPUTimeThreadSampling* JfrCPUTimeThreadSampling::create() {
+ _instance = new JfrCPUTimeThreadSampling();
+ return _instance;
+}
+
+void JfrCPUTimeThreadSampling::destroy() {
+ delete _instance;
+ _instance = nullptr;
+}
+
+void JfrCPUTimeThreadSampling::set_rate(double rate, bool auto_adapt) {
+ if (rate != 0) {
+ warn();
+ }
+}
+
+void JfrCPUTimeThreadSampling::on_javathread_create(JavaThread* thread) {
+}
+
+void JfrCPUTimeThreadSampling::on_javathread_terminate(JavaThread* thread) {
+}
+
+#endif // defined(LINUX) && defined(INCLUDE_JFR)
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp b/src/hotspot/share/jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp
new file mode 100644
index 00000000000..7c0545f4772
--- /dev/null
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2025 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_PERIODIC_SAMPLING_JFRCPUTIMETHREADSAMPLER_HPP
+#define SHARE_JFR_PERIODIC_SAMPLING_JFRCPUTIMETHREADSAMPLER_HPP
+
+#include "jfr/utilities/jfrAllocation.hpp"
+
+class JavaThread;
+
+#if defined(LINUX)
+
+#include "jfr/periodic/sampling/jfrSampleRequest.hpp"
+#include "jfr/utilities/jfrTypes.hpp"
+
+struct JfrCPUTimeSampleRequest {
+ JfrSampleRequest _request;
+ Tickspan _cpu_time_period;
+
+ JfrCPUTimeSampleRequest() {}
+};
+
+// Fixed size async-signal-safe SPSC linear queue backed by an array.
+// Designed to be only used under lock and read linearly
+class JfrCPUTimeTraceQueue {
+
+ // the default queue capacity, scaled if the sampling period is smaller than 10ms
+ // when the thread is started
+ static const u4 CPU_TIME_QUEUE_CAPACITY = 500;
+
+ JfrCPUTimeSampleRequest* _data;
+ u4 _capacity;
+ // next unfilled index
+ volatile u4 _head;
+
+ volatile u4 _lost_samples;
+
+public:
+ JfrCPUTimeTraceQueue(u4 capacity);
+
+ ~JfrCPUTimeTraceQueue();
+
+ // signal safe, but can't be interleaved with dequeue
+ bool enqueue(JfrCPUTimeSampleRequest& trace);
+
+ JfrCPUTimeSampleRequest& at(u4 index);
+
+ u4 size() const;
+
+ void set_size(u4 size);
+
+ u4 capacity() const;
+
+ // deletes all samples in the queue
+ void set_capacity(u4 capacity);
+
+ bool is_empty() const;
+
+ u4 lost_samples() const;
+
+ void increment_lost_samples();
+
+ // returns the previous lost samples count
+ u4 get_and_reset_lost_samples();
+
+ void resize(u4 capacity);
+
+ void resize_for_period(u4 period_millis);
+
+ void clear();
+
+};
+
+
+class JfrCPUSamplerThread;
+
+class JfrCPUTimeThreadSampling : public JfrCHeapObj {
+ friend class JfrRecorder;
+ private:
+
+ JfrCPUSamplerThread* _sampler;
+
+ void create_sampler(double rate, bool auto_adapt);
+ void set_rate_value(double rate, bool auto_adapt);
+
+ JfrCPUTimeThreadSampling();
+ ~JfrCPUTimeThreadSampling();
+
+ static JfrCPUTimeThreadSampling& instance();
+ static JfrCPUTimeThreadSampling* create();
+ static void destroy();
+
+ void update_run_state(double rate, bool auto_adapt);
+
+ public:
+ static void set_rate(double rate, bool auto_adapt);
+
+ static void on_javathread_create(JavaThread* thread);
+ static void on_javathread_terminate(JavaThread* thread);
+ void handle_timer_signal(siginfo_t* info, void* context);
+
+ static void send_empty_event(const JfrTicks& start_time, traceid tid, Tickspan cpu_time_period);
+ static void send_event(const JfrTicks& start_time, traceid sid, traceid tid, Tickspan cpu_time_period, bool biased);
+ static void send_lost_event(const JfrTicks& time, traceid tid, s4 lost_samples);
+
+ static void trigger_async_processing_of_cpu_time_jfr_requests();
+};
+
+#else
+
+// a basic implementation on other platforms that
+// emits warnings
+
+class JfrCPUTimeThreadSampling : public JfrCHeapObj {
+ friend class JfrRecorder;
+private:
+ static JfrCPUTimeThreadSampling& instance();
+ static JfrCPUTimeThreadSampling* create();
+ static void destroy();
+
+ public:
+ static void set_rate(double rate, bool auto_adapt);
+
+ static void on_javathread_create(JavaThread* thread);
+ static void on_javathread_terminate(JavaThread* thread);
+};
+
+#endif // defined(LINUX)
+
+
+#endif // SHARE_JFR_PERIODIC_SAMPLING_JFRCPUTIMETHREADSAMPLER_HPP
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrSampleMonitor.hpp b/src/hotspot/share/jfr/periodic/sampling/jfrSampleMonitor.hpp
new file mode 100644
index 00000000000..a9e0b1728a3
--- /dev/null
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrSampleMonitor.hpp
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_JFR_PERIODIC_SAMPLING_JFRSAMPLEMONITOR_HPP
+#define SHARE_JFR_PERIODIC_SAMPLING_JFRSAMPLEMONITOR_HPP
+
+#include "jfr/periodic/sampling/jfrSampleRequest.hpp"
+#include "jfr/utilities/jfrTime.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/mutex.hpp"
+
+class JfrSampleMonitor : public StackObj {
+ private:
+ JfrThreadLocal* const _tl;
+ Monitor* const _sample_monitor;
+ mutable bool _waiting;
+ public:
+ JfrSampleMonitor(JfrThreadLocal* tl) :
+ _tl(tl), _sample_monitor(tl->sample_monitor()), _waiting(false) {
+ assert(tl != nullptr, "invariant");
+ assert(_sample_monitor != nullptr, "invariant");
+ _sample_monitor->lock_without_safepoint_check();
+ }
+
+ bool is_waiting() const {
+ assert_lock_strong(_sample_monitor);
+ _waiting = _tl->sample_state() == WAITING_FOR_NATIVE_SAMPLE;
+ return _waiting;
+ }
+
+ void install_java_sample_request() {
+ assert_lock_strong(_sample_monitor);
+ assert(_waiting, "invariant");
+ assert(_tl->sample_state() == WAITING_FOR_NATIVE_SAMPLE, "invariant");
+ JfrSampleRequest request;
+ request._sample_ticks = JfrTicks::now();
+ _tl->set_sample_request(request);
+ _tl->set_sample_state(JAVA_SAMPLE);
+ _sample_monitor->notify_all();
+ }
+
+ ~JfrSampleMonitor() {
+ assert_lock_strong(_sample_monitor);
+ if (!_waiting) {
+ _tl->set_sample_state(NO_SAMPLE);
+ _sample_monitor->notify_all();
+ }
+ _sample_monitor->unlock();
+ }
+};
+
+#endif // SHARE_JFR_PERIODIC_SAMPLING_JFRSAMPLEMONITOR_HPP
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.cpp b/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.cpp
index f8e63e2e344..7049df0198b 100644
--- a/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.cpp
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.cpp
@@ -24,6 +24,7 @@
#include "asm/codeBuffer.hpp"
#include "interpreter/interpreter.hpp"
#include "jfr/periodic/sampling/jfrSampleRequest.hpp"
+#include "jfr/utilities/jfrTime.hpp"
#include "runtime/continuationEntry.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/javaThread.inline.hpp"
@@ -171,7 +172,7 @@ static bool build(JfrSampleRequest& request, intptr_t* fp, JavaThread* jt) {
assert(request._sample_sp != nullptr, "invariant");
assert(request._sample_pc != nullptr, "invariant");
assert(jt != nullptr, "invariant");
- assert(jt->thread_state() == _thread_in_Java, "invariant");
+ assert(jt->thread_state() == _thread_in_Java || jt->thread_state() == _thread_in_native, "invariant");
// 1. Interpreter frame?
if (is_interpreter(request)) {
@@ -303,3 +304,33 @@ JfrSampleResult JfrSampleRequestBuilder::build_java_sample_request(const void* u
}
return set_biased_java_sample(request, tl, jt);
}
+
+
+// A biased sample request is denoted by an empty bcp and an empty pc.
+static inline void set_cpu_time_biased_sample(JfrSampleRequest& request, JavaThread* jt) {
+ if (request._sample_bcp != nullptr) {
+ request._sample_bcp = nullptr;
+ }
+ assert(request._sample_bcp == nullptr, "invariant");
+ request._sample_pc = nullptr;
+}
+
+void JfrSampleRequestBuilder::build_cpu_time_sample_request(JfrSampleRequest& request,
+ void* ucontext,
+ JavaThread* jt,
+ JfrThreadLocal* tl,
+ JfrTicks& now) {
+ assert(jt != nullptr, "invariant");
+ request._sample_ticks = now;
+
+ // Prioritize the ljf, if one exists.
+ request._sample_sp = jt->last_Java_sp();
+ if (request._sample_sp == nullptr || !build_from_ljf(request, tl, jt)) {
+ intptr_t* fp;
+ request._sample_pc = os::fetch_frame_from_context(ucontext, reinterpret_cast(&request._sample_sp), &fp);
+ assert(sp_in_stack(request, jt), "invariant");
+ if (!build(request, fp, jt)) {
+ set_cpu_time_biased_sample(request, jt);
+ }
+ }
+}
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.hpp b/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.hpp
index 8cc2b66aa9e..20e737e0cbf 100644
--- a/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.hpp
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrSampleRequest.hpp
@@ -48,10 +48,11 @@ enum JfrSampleResult {
};
enum JfrSampleRequestType {
- NO_SAMPLE = 0,
- NATIVE_SAMPLE = 1,
- JAVA_SAMPLE = 2,
- NOF_SAMPLE_TYPES
+ NO_SAMPLE,
+ JAVA_SAMPLE,
+ NATIVE_SAMPLE,
+ WAITING_FOR_NATIVE_SAMPLE,
+ NOF_SAMPLE_STATES
};
struct JfrSampleRequest {
@@ -80,6 +81,11 @@ class JfrSampleRequestBuilder : AllStatic {
static JfrSampleResult build_java_sample_request(const void* ucontext,
JfrThreadLocal* tl,
JavaThread* jt);
+ static void build_cpu_time_sample_request(JfrSampleRequest &request,
+ void* ucontext,
+ JavaThread* jt,
+ JfrThreadLocal* tl,
+ JfrTicks& now);
};
#endif // SHARE_JFR_PERIODIC_SAMPLING_JFRSAMPLEREQUEST_HPP
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp
index 3efa0b0d581..4c44c43772d 100644
--- a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp
@@ -24,6 +24,7 @@
#include "jfr/metadata/jfrSerializer.hpp"
#include "jfr/recorder/service/jfrOptionSet.hpp"
+#include "jfr/periodic/sampling/jfrSampleMonitor.hpp"
#include "jfr/periodic/sampling/jfrSampleRequest.hpp"
#include "jfr/periodic/sampling/jfrThreadSampling.hpp"
#include "jfr/periodic/sampling/jfrThreadSampler.hpp"
@@ -230,38 +231,41 @@ void JfrSamplerThread::task_stacktrace(JfrSampleRequestType type, JavaThread** l
JavaThread* start = nullptr;
elapsedTimer sample_time;
sample_time.start();
- {
- MutexLocker tlock(Threads_lock);
- ThreadsListHandle tlh;
- // Resolve a sample session relative start position index into the thread list array.
- // In cases where the last sampled thread is null or not-null but stale, find_index() returns -1.
- _cur_index = tlh.list()->find_index_of_JavaThread(*last_thread);
- JavaThread* current = _cur_index != -1 ? *last_thread : nullptr;
+ ThreadsListHandle tlh;
+ // Resolve a sample session relative start position index into the thread list array.
+ // In cases where the last sampled thread is null or not-null but stale, find_index() returns -1.
+ _cur_index = tlh.list()->find_index_of_JavaThread(*last_thread);
+ JavaThread* current = _cur_index != -1 ? *last_thread : nullptr;
- while (num_samples < sample_limit) {
- current = next_thread(tlh.list(), start, current);
- if (current == nullptr) {
- break;
- }
- if (is_excluded(current)) {
- continue;
- }
- if (start == nullptr) {
- start = current; // remember the thread where we started to attempt sampling
- }
- bool success;
- if (JAVA_SAMPLE == type) {
- success = sample_java_thread(current);
- } else {
- assert(type == NATIVE_SAMPLE, "invariant");
- success = sample_native_thread(current);
- }
- if (success) {
- num_samples++;
- }
+ while (num_samples < sample_limit) {
+ current = next_thread(tlh.list(), start, current);
+ if (current == nullptr) {
+ break;
+ }
+ if (is_excluded(current)) {
+ continue;
+ }
+ if (start == nullptr) {
+ start = current; // remember the thread where we started to attempt sampling
+ }
+ bool success;
+ if (JAVA_SAMPLE == type) {
+ success = sample_java_thread(current);
+ } else {
+ assert(type == NATIVE_SAMPLE, "invariant");
+ success = sample_native_thread(current);
+ }
+ if (success) {
+ num_samples++;
+ }
+ if (SafepointSynchronize::is_at_safepoint()) {
+ // For _thread_in_native, we cannot get the Threads_lock.
+ // For _thread_in_Java, well, there are none.
+ break;
}
- *last_thread = current; // remember the thread we last attempted to sample
}
+
+ *last_thread = current; // remember the thread we last attempted to sample
sample_time.stop();
log_trace(jfr)("JFR thread sampling done in %3.7f secs with %d java %d native samples",
sample_time.seconds(), type == JAVA_SAMPLE ? num_samples : 0, type == NATIVE_SAMPLE ? num_samples : 0);
@@ -338,17 +342,32 @@ bool JfrSamplerThread::sample_native_thread(JavaThread* jt) {
SafepointMechanism::arm_local_poll_release(jt);
- // Barriers needed to keep the next read of thread state from floating up.
- if (UseSystemMemoryBarrier) {
- SystemMemoryBarrier::emit();
- } else {
- OrderAccess::storeload();
+ // Take the Threads_lock for two purposes:
+ // 1) Avoid sampling through a safepoint which could result
+ // in touching oops in case of virtual threads.
+ // 2) Prevent JFR from issuing an epoch rotation while the sampler thread
+ // is actively processing a thread in native, as both threads are now
+ // outside the safepoint protocol.
+
+ // OrderAccess::fence() as part of acquiring the lock prevents loads from floating up.
+ JfrMutexTryLock threads_lock(Threads_lock);
+
+ if (!threads_lock.acquired() || !jt->has_last_Java_frame()) {
+ // Remove the native sample request and release the potentially waiting thread.
+ JfrSampleMonitor jsm(tl);
+ return false;
}
- if (jt->thread_state() != _thread_in_native || !jt->has_last_Java_frame()) {
- MonitorLocker lock(tl->sample_monitor(), Monitor::_no_safepoint_check_flag);
- tl->set_sample_state(NO_SAMPLE);
- lock.notify_all();
+ if (jt->thread_state() != _thread_in_native) {
+ assert_lock_strong(Threads_lock);
+ JfrSampleMonitor jsm(tl);
+ if (jsm.is_waiting()) {
+ // The thread has already returned from native,
+ // now in _thread_in_vm and is waiting to be sampled.
+ // Convert the native sample request into a java sample request
+ // and let the thread process the ljf on its own.
+ jsm.install_java_sample_request();
+ }
return false;
}
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.cpp b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.cpp
index 9bae25bcb3c..ddc9d59b295 100644
--- a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.cpp
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.cpp
@@ -28,6 +28,8 @@
#include "code/nmethod.hpp"
#include "interpreter/interpreter.hpp"
#include "jfr/jfrEvents.hpp"
+#include "jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp"
+#include "jfr/periodic/sampling/jfrSampleMonitor.hpp"
#include "jfr/periodic/sampling/jfrSampleRequest.hpp"
#include "jfr/periodic/sampling/jfrThreadSampling.hpp"
#include "jfr/recorder/stacktrace/jfrStackTrace.hpp"
@@ -160,7 +162,7 @@ static inline bool is_valid(const PcDesc* pc_desc) {
return pc_desc != nullptr && pc_desc->scope_decode_offset() != DebugInformationRecorder::serialized_null;
}
-static bool compute_top_frame(const JfrSampleRequest& request, frame& top_frame, bool& in_continuation, JavaThread* jt) {
+static bool compute_top_frame(const JfrSampleRequest& request, frame& top_frame, bool& in_continuation, JavaThread* jt, bool& biased) {
assert(jt != nullptr, "invariant");
if (!jt->has_last_Java_frame()) {
@@ -177,6 +179,7 @@ static bool compute_top_frame(const JfrSampleRequest& request, frame& top_frame,
// A biased sample is requested or no code blob.
top_frame = jt->last_frame();
in_continuation = is_in_continuation(top_frame, jt);
+ biased = true;
return true;
}
@@ -226,6 +229,8 @@ static bool compute_top_frame(const JfrSampleRequest& request, frame& top_frame,
assert(!stream.current()->is_safepoint_blob_frame(), "invariant");
+ biased = true;
+
// Search the first frame that is above the sampled sp.
for (; !stream.is_done(); stream.next()) {
frame* const current = stream.current();
@@ -249,6 +254,7 @@ static bool compute_top_frame(const JfrSampleRequest& request, frame& top_frame,
const PcDesc* const pc_desc = get_pc_desc(sampled_nm, sampled_pc);
if (is_valid(pc_desc)) {
current->adjust_pc(pc_desc->real_pc(sampled_nm));
+ biased = false;
}
}
}
@@ -269,8 +275,9 @@ static void record_thread_in_java(const JfrSampleRequest& request, const JfrTick
assert(current != nullptr, "invariant");
frame top_frame;
+ bool biased = false;
bool in_continuation;
- if (!compute_top_frame(request, top_frame, in_continuation, jt)) {
+ if (!compute_top_frame(request, top_frame, in_continuation, jt, biased)) {
return;
}
@@ -292,6 +299,42 @@ static void record_thread_in_java(const JfrSampleRequest& request, const JfrTick
}
}
+#ifdef LINUX
+static void record_cpu_time_thread(const JfrCPUTimeSampleRequest& request, const JfrTicks& now, const JfrThreadLocal* tl, JavaThread* jt, Thread* current) {
+ assert(jt != nullptr, "invariant");
+ assert(tl != nullptr, "invariant");
+ assert(current != nullptr, "invariant");
+ frame top_frame;
+ bool biased = false;
+ bool in_continuation = false;
+ bool could_compute_top_frame = compute_top_frame(request._request, top_frame, in_continuation, jt, biased);
+ const traceid tid = in_continuation ? tl->vthread_id_with_epoch_update(jt) : JfrThreadLocal::jvm_thread_id(jt);
+
+ if (!could_compute_top_frame) {
+ JfrCPUTimeThreadSampling::send_empty_event(request._request._sample_ticks, tid, request._cpu_time_period);
+ return;
+ }
+ traceid sid;
+ {
+ ResourceMark rm(current);
+ JfrStackTrace stacktrace;
+ if (!stacktrace.record(jt, top_frame, in_continuation, request._request)) {
+ // Unable to record stacktrace. Fail.
+ JfrCPUTimeThreadSampling::send_empty_event(request._request._sample_ticks, tid, request._cpu_time_period);
+ return;
+ }
+ sid = JfrStackTraceRepository::add(stacktrace);
+ }
+ assert(sid != 0, "invariant");
+
+
+ JfrCPUTimeThreadSampling::send_event(request._request._sample_ticks, sid, tid, request._cpu_time_period, biased);
+ if (current == jt) {
+ send_safepoint_latency_event(request._request, now, sid, jt);
+ }
+}
+#endif
+
static void drain_enqueued_requests(const JfrTicks& now, JfrThreadLocal* tl, JavaThread* jt, Thread* current) {
assert(tl != nullptr, "invariant");
assert(jt != nullptr, "invariant");
@@ -307,23 +350,48 @@ static void drain_enqueued_requests(const JfrTicks& now, JfrThreadLocal* tl, Jav
assert(!tl->has_enqueued_requests(), "invariant");
}
-class SampleMonitor : public StackObj {
- private:
- JfrThreadLocal* const _tl;
- Monitor* const _sample_monitor;
- public:
- SampleMonitor(JfrThreadLocal* tl) : _tl(tl), _sample_monitor(tl->sample_monitor()) {
- assert(tl != nullptr, "invariant");
- assert(_sample_monitor != nullptr, "invariant");
- _sample_monitor->lock_without_safepoint_check();
+static void drain_enqueued_cpu_time_requests(const JfrTicks& now, JfrThreadLocal* tl, JavaThread* jt, Thread* current, bool lock) {
+ assert(tl != nullptr, "invariant");
+ assert(jt != nullptr, "invariant");
+ assert(current != nullptr, "invariant");
+#ifdef LINUX
+ tl->set_do_async_processing_of_cpu_time_jfr_requests(false);
+ if (lock) {
+ tl->acquire_cpu_time_jfr_dequeue_lock();
}
- ~SampleMonitor() {
- assert_lock_strong(_sample_monitor);
- _tl->set_sample_state(NO_SAMPLE);
- _sample_monitor->notify_all();
- _sample_monitor->unlock();
+ JfrCPUTimeTraceQueue& queue = tl->cpu_time_jfr_queue();
+ for (u4 i = 0; i < queue.size(); i++) {
+ record_cpu_time_thread(queue.at(i), now, tl, jt, current);
}
-};
+ queue.clear();
+ assert(queue.is_empty(), "invariant");
+ tl->set_has_cpu_time_jfr_requests(false);
+ if (queue.lost_samples() > 0) {
+ JfrCPUTimeThreadSampling::send_lost_event( now, JfrThreadLocal::thread_id(jt), queue.get_and_reset_lost_samples());
+ }
+ if (lock) {
+ tl->release_cpu_time_jfr_queue_lock();
+ }
+#endif
+}
+
+// Entry point for a thread that has been sampled in native code and has a pending JFR CPU time request.
+void JfrThreadSampling::process_cpu_time_request(JavaThread* jt, JfrThreadLocal* tl, Thread* current, bool lock) {
+ assert(jt != nullptr, "invariant");
+
+ const JfrTicks now = JfrTicks::now();
+ drain_enqueued_cpu_time_requests(now, tl, jt, current, lock);
+}
+
+static void drain_all_enqueued_requests(const JfrTicks& now, JfrThreadLocal* tl, JavaThread* jt, Thread* current) {
+ assert(tl != nullptr, "invariant");
+ assert(jt != nullptr, "invariant");
+ assert(current != nullptr, "invariant");
+ drain_enqueued_requests(now, tl, jt, current);
+ if (tl->has_cpu_time_jfr_requests()) {
+ drain_enqueued_cpu_time_requests(now, tl, jt, current, true);
+ }
+}
// Only entered by the JfrSampler thread.
bool JfrThreadSampling::process_native_sample_request(JfrThreadLocal* tl, JavaThread* jt, Thread* sampler_thread) {
@@ -334,7 +402,9 @@ bool JfrThreadSampling::process_native_sample_request(JfrThreadLocal* tl, JavaTh
assert(tl == jt->jfr_thread_local(), "invariant");
assert(jt != sampler_thread, "only asynchronous processing of native samples");
assert(jt->has_last_Java_frame(), "invariant");
- assert(tl->sample_state() == NATIVE_SAMPLE, "invariant");
+ assert(tl->sample_state() >= NATIVE_SAMPLE, "invariant");
+
+ assert_lock_strong(Threads_lock);
const JfrTicks start_time = JfrTicks::now();
@@ -342,7 +412,7 @@ bool JfrThreadSampling::process_native_sample_request(JfrThreadLocal* tl, JavaTh
traceid sid;
{
- SampleMonitor sm(tl);
+ JfrSampleMonitor sm(tl);
// Because the thread was in native, it is in a walkable state, because
// it will hit a safepoint poll on the way back from native. To ensure timely
@@ -384,14 +454,19 @@ void JfrThreadSampling::process_sample_request(JavaThread* jt) {
for (;;) {
const int sample_state = tl->sample_state();
if (sample_state == NATIVE_SAMPLE) {
+ tl->set_sample_state(WAITING_FOR_NATIVE_SAMPLE);
// Wait until stack trace is processed.
ml.wait();
} else if (sample_state == JAVA_SAMPLE) {
tl->enqueue_request();
+ } else if (sample_state == WAITING_FOR_NATIVE_SAMPLE) {
+ // Handle spurious wakeups. Again wait until stack trace is processed.
+ ml.wait();
} else {
// State has been processed.
break;
}
}
- drain_enqueued_requests(now, tl, jt, jt);
+ drain_all_enqueued_requests(now, tl, jt, jt);
}
+
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.hpp b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.hpp
index d4c4bc0d873..3c4d1a126b0 100644
--- a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.hpp
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampling.hpp
@@ -33,8 +33,10 @@ class Thread;
class JfrThreadSampling : AllStatic {
friend class JfrSamplerThread;
+ friend class JfrCPUSamplerThread;
private:
static bool process_native_sample_request(JfrThreadLocal* tl, JavaThread* jt, Thread* sampler_thread);
+ static void process_cpu_time_request(JavaThread* jt, JfrThreadLocal* tl, Thread* current, bool lock);
public:
static void process_sample_request(JavaThread* jt);
};
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp
index 7736d3f4565..b0f4461a82c 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp
@@ -497,15 +497,10 @@ typedef CompositeOperation WriteRelease
typedef VirtualThreadLocalCheckpointWriteOp VirtualThreadLocalCheckpointOperation;
typedef MutexedWriteOp VirtualThreadLocalWriteOperation;
-void JfrCheckpointManager::begin_epoch_shift() {
- assert(SafepointSynchronize::is_at_safepoint(), "invariant");
- JfrTraceIdEpoch::begin_epoch_shift();
-}
-
-void JfrCheckpointManager::end_epoch_shift() {
+void JfrCheckpointManager::shift_epoch() {
assert(SafepointSynchronize::is_at_safepoint(), "invariant");
DEBUG_ONLY(const u1 current_epoch = JfrTraceIdEpoch::current();)
- JfrTraceIdEpoch::end_epoch_shift();
+ JfrTraceIdEpoch::shift_epoch();
assert(current_epoch != JfrTraceIdEpoch::current(), "invariant");
JfrStringPool::on_epoch_shift();
}
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp
index 53fa064e267..f9f8f1c26cf 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp
@@ -96,8 +96,7 @@ class JfrCheckpointManager : public JfrCHeapObj {
void clear_type_set();
void write_type_set();
- void begin_epoch_shift();
- void end_epoch_shift();
+ void shift_epoch();
static void on_unloading_classes();
void on_rotation();
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp
index 8c2b8cdd0ec..a4ada594700 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp
@@ -31,7 +31,7 @@
/*
* The epoch generation is the range [1-32767].
*
- * When the epoch value is stored in a thread object,
+ * When the epoch value is stored in a vthread object,
* the most significant bit of the u2 is used to denote
* thread exclusion, i.e 1 << 15 == 32768 denotes exclusion.
*/
@@ -39,32 +39,17 @@ u2 JfrTraceIdEpoch::_generation = 0;
JfrSignal JfrTraceIdEpoch::_tag_state;
bool JfrTraceIdEpoch::_method_tracer_state = false;
bool JfrTraceIdEpoch::_epoch_state = false;
-bool JfrTraceIdEpoch::_synchronizing = false;
static constexpr const u2 epoch_generation_overflow = excluded_bit;
-void JfrTraceIdEpoch::begin_epoch_shift() {
+void JfrTraceIdEpoch::shift_epoch() {
assert(SafepointSynchronize::is_at_safepoint(), "invariant");
- _synchronizing = true;
- OrderAccess::fence();
-}
-
-void JfrTraceIdEpoch::end_epoch_shift() {
- assert(SafepointSynchronize::is_at_safepoint(), "invariant");
- assert(_synchronizing, "invariant");
_epoch_state = !_epoch_state;
- ++_generation;
- if (epoch_generation_overflow == _generation) {
+ if (++_generation == epoch_generation_overflow) {
_generation = 1;
}
assert(_generation != 0, "invariant");
assert(_generation < epoch_generation_overflow, "invariant");
- OrderAccess::storestore();
- _synchronizing = false;
-}
-
-bool JfrTraceIdEpoch::is_synchronizing() {
- return Atomic::load_acquire(&_synchronizing);
}
void JfrTraceIdEpoch::set_method_tracer_tag_state() {
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp
index 10ea9643971..9e2d2f0708a 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp
@@ -26,7 +26,6 @@
#define SHARE_JFR_RECORDER_CHECKPOINT_TYPES_TRACEID_JFRTRACEIDEPOCH_HPP
#include "jfr/utilities/jfrSignal.hpp"
-#include "jfr/utilities/jfrTypes.hpp"
#include "memory/allStatic.hpp"
#define BIT 1
@@ -41,16 +40,17 @@
#define EPOCH_0_METHOD_AND_CLASS_BITS (METHOD_AND_CLASS_BITS << EPOCH_0_SHIFT)
#define EPOCH_1_METHOD_AND_CLASS_BITS (METHOD_AND_CLASS_BITS << EPOCH_1_SHIFT)
- // Epoch alternation on each rotation allow for concurrent tagging.
- // The epoch shift happens only during a safepoint.
- //
- // _synchronizing is a transition state, the purpose of which is to
- // have JavaThreads that run _thread_in_native (i.e. Compiler threads)
- // respect the current epoch shift in-progress during the safepoint.
- //
- // _changed_tag_state == true signals an incremental modification to artifact tagging
- // (klasses, methods, CLDs, etc), purpose of which is to trigger collection of artifacts.
- //
+/*
+ * An epoch shift or alternation on each rotation enables concurrent tagging.
+ * The epoch shift happens only during a safepoint.
+ *
+ * _generation - mainly used with virtual threads, but also for the generational string pool in Java.
+ * _tag_state - signals an incremental modification to artifact tagging (klasses, methods, CLDs, etc)
+ * purpose of which is to trigger a collection of artifacts.
+ * _method_tracer_state - a special notification state only used with method timing and tracing.
+ * _epoch_state - the fundamental binary epoch state that shifts on each rotation during a safepoint.
+ */
+
class JfrTraceIdEpoch : AllStatic {
friend class JfrCheckpointManager;
private:
@@ -58,10 +58,8 @@ class JfrTraceIdEpoch : AllStatic {
static JfrSignal _tag_state;
static bool _method_tracer_state;
static bool _epoch_state;
- static bool _synchronizing;
- static void begin_epoch_shift();
- static void end_epoch_shift();
+ static void shift_epoch();
public:
static bool epoch() {
diff --git a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
index 384305862ca..dd75cb2929f 100644
--- a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
+++ b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
@@ -29,6 +29,7 @@
#include "jfr/jni/jfrJavaSupport.hpp"
#include "jfr/leakprofiler/sampling/objectSampler.hpp"
#include "jfr/periodic/jfrOSInterface.hpp"
+#include "jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp"
#include "jfr/periodic/sampling/jfrThreadSampler.hpp"
#include "jfr/recorder/jfrRecorder.hpp"
#include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
@@ -304,6 +305,9 @@ bool JfrRecorder::create_components() {
if (!create_thread_sampler()) {
return false;
}
+ if (!create_cpu_time_thread_sampling()) {
+ return false;
+ }
if (!create_event_throttler()) {
return false;
}
@@ -318,6 +322,7 @@ static JfrStackTraceRepository* _stack_trace_repository;
static JfrStringPool* _stringpool = nullptr;
static JfrOSInterface* _os_interface = nullptr;
static JfrThreadSampler* _thread_sampler = nullptr;
+static JfrCPUTimeThreadSampling* _cpu_time_thread_sampling = nullptr;
static JfrCheckpointManager* _checkpoint_manager = nullptr;
bool JfrRecorder::create_java_event_writer() {
@@ -390,6 +395,12 @@ bool JfrRecorder::create_thread_sampler() {
return _thread_sampler != nullptr;
}
+bool JfrRecorder::create_cpu_time_thread_sampling() {
+ assert(_cpu_time_thread_sampling == nullptr, "invariant");
+ _cpu_time_thread_sampling = JfrCPUTimeThreadSampling::create();
+ return _cpu_time_thread_sampling != nullptr;
+}
+
bool JfrRecorder::create_event_throttler() {
return JfrEventThrottler::create();
}
@@ -428,6 +439,10 @@ void JfrRecorder::destroy_components() {
JfrThreadSampler::destroy();
_thread_sampler = nullptr;
}
+ if (_cpu_time_thread_sampling != nullptr) {
+ JfrCPUTimeThreadSampling::destroy();
+ _cpu_time_thread_sampling = nullptr;
+ }
JfrEventThrottler::destroy();
}
diff --git a/src/hotspot/share/jfr/recorder/jfrRecorder.hpp b/src/hotspot/share/jfr/recorder/jfrRecorder.hpp
index b917904c5fb..3099c8ad344 100644
--- a/src/hotspot/share/jfr/recorder/jfrRecorder.hpp
+++ b/src/hotspot/share/jfr/recorder/jfrRecorder.hpp
@@ -54,6 +54,7 @@ class JfrRecorder : public JfrCHeapObj {
static bool create_storage();
static bool create_stringpool();
static bool create_thread_sampler();
+ static bool create_cpu_time_thread_sampling();
static bool create_event_throttler();
static bool create_components();
static void destroy_components();
diff --git a/src/hotspot/share/jfr/recorder/service/jfrEventThrottler.cpp b/src/hotspot/share/jfr/recorder/service/jfrEventThrottler.cpp
index 0befaae7751..f660a01c04c 100644
--- a/src/hotspot/share/jfr/recorder/service/jfrEventThrottler.cpp
+++ b/src/hotspot/share/jfr/recorder/service/jfrEventThrottler.cpp
@@ -34,6 +34,7 @@ constexpr static const JfrSamplerParams _disabled_params = {
false // reconfigure
};
+static JfrEventThrottler* _disabled_cpu_time_sample_throttler = nullptr;
static JfrEventThrottler* _object_allocation_throttler = nullptr;
static JfrEventThrottler* _safepoint_latency_throttler = nullptr;
@@ -48,6 +49,9 @@ JfrEventThrottler::JfrEventThrottler(JfrEventId event_id) :
_update(false) {}
bool JfrEventThrottler::create() {
+ assert(_disabled_cpu_time_sample_throttler == nullptr, "invariant");
+ _disabled_cpu_time_sample_throttler = new JfrEventThrottler(JfrCPUTimeSampleEvent);
+ _disabled_cpu_time_sample_throttler->_disabled = true;
assert(_object_allocation_throttler == nullptr, "invariant");
_object_allocation_throttler = new JfrEventThrottler(JfrObjectAllocationSampleEvent);
if (_object_allocation_throttler == nullptr || !_object_allocation_throttler->initialize()) {
@@ -59,6 +63,8 @@ bool JfrEventThrottler::create() {
}
void JfrEventThrottler::destroy() {
+ delete _disabled_cpu_time_sample_throttler;
+ _disabled_cpu_time_sample_throttler = nullptr;
delete _object_allocation_throttler;
_object_allocation_throttler = nullptr;
delete _safepoint_latency_throttler;
@@ -69,15 +75,19 @@ void JfrEventThrottler::destroy() {
// and another for the SamplingLatency event.
// When introducing many more throttlers, consider adding a lookup map keyed by event id.
JfrEventThrottler* JfrEventThrottler::for_event(JfrEventId event_id) {
+ assert(_disabled_cpu_time_sample_throttler != nullptr, "Disabled CPU time throttler has not been properly initialized");
assert(_object_allocation_throttler != nullptr, "ObjectAllocation throttler has not been properly initialized");
assert(_safepoint_latency_throttler != nullptr, "SafepointLatency throttler has not been properly initialized");
- assert(event_id == JfrObjectAllocationSampleEvent || event_id == JfrSafepointLatencyEvent, "Event type has an unconfigured throttler");
+ assert(event_id == JfrObjectAllocationSampleEvent || event_id == JfrSafepointLatencyEvent || event_id == JfrCPUTimeSampleEvent, "Event type has an unconfigured throttler");
if (event_id == JfrObjectAllocationSampleEvent) {
return _object_allocation_throttler;
}
if (event_id == JfrSafepointLatencyEvent) {
return _safepoint_latency_throttler;
}
+ if (event_id == JfrCPUTimeSampleEvent) {
+ return _disabled_cpu_time_sample_throttler;
+ }
return nullptr;
}
diff --git a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp
index 07fe019c9af..7d1d7ac0a05 100644
--- a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp
+++ b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp
@@ -484,13 +484,12 @@ void JfrRecorderService::invoke_safepoint_clear() {
void JfrRecorderService::safepoint_clear() {
assert(SafepointSynchronize::is_at_safepoint(), "invariant");
- _checkpoint_manager.begin_epoch_shift();
_storage.clear();
_checkpoint_manager.notify_threads();
_chunkwriter.set_time_stamp();
JfrDeprecationManager::on_safepoint_clear();
JfrStackTraceRepository::clear();
- _checkpoint_manager.end_epoch_shift();
+ _checkpoint_manager.shift_epoch();
}
void JfrRecorderService::post_safepoint_clear() {
@@ -593,14 +592,13 @@ void JfrRecorderService::invoke_safepoint_write() {
void JfrRecorderService::safepoint_write() {
assert(SafepointSynchronize::is_at_safepoint(), "invariant");
- _checkpoint_manager.begin_epoch_shift();
JfrStackTraceRepository::clear_leak_profiler();
_checkpoint_manager.on_rotation();
_storage.write_at_safepoint();
_chunkwriter.set_time_stamp();
JfrDeprecationManager::on_safepoint_write();
write_stacktrace(_stack_trace_repository, _chunkwriter, true);
- _checkpoint_manager.end_epoch_shift();
+ _checkpoint_manager.shift_epoch();
}
void JfrRecorderService::post_safepoint_write() {
diff --git a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp
index a9e39094f79..dc28818b0f9 100644
--- a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp
+++ b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp
@@ -44,8 +44,6 @@
static int generation_offset = invalid_offset;
static jobject string_pool = nullptr;
-static unsigned short generation = 0;
-
static bool setup_string_pool_offsets(TRAPS) {
const char class_name[] = "jdk/jfr/internal/StringPool";
Symbol* const k_sym = SymbolTable::new_symbol(class_name);
@@ -281,9 +279,8 @@ void JfrStringPool::register_full(BufferPtr buffer, Thread* thread) {
void JfrStringPool::on_epoch_shift() {
assert(SafepointSynchronize::is_at_safepoint(), "invariant");
- assert(!JfrTraceIdEpoch::is_synchronizing(), "invariant");
assert(string_pool != nullptr, "invariant");
oop mirror = JfrJavaSupport::resolve_non_null(string_pool);
assert(mirror != nullptr, "invariant");
- mirror->short_field_put(generation_offset, generation++);
+ mirror->short_field_put(generation_offset, JfrTraceIdEpoch::epoch_generation());
}
diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp
index 503aa85e02f..4b805a98a32 100644
--- a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp
+++ b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp
@@ -26,6 +26,7 @@
#include "jfr/jni/jfrJavaSupport.hpp"
#include "jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp"
#include "jfr/periodic/jfrThreadCPULoadEvent.hpp"
+#include "jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp"
#include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
#include "jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.inline.hpp"
#include "jfr/recorder/jfrRecorder.hpp"
@@ -78,7 +79,15 @@ JfrThreadLocal::JfrThreadLocal() :
_enqueued_requests(false),
_vthread(false),
_notified(false),
- _dead(false) {
+ _dead(false)
+#ifdef LINUX
+ ,_cpu_timer(nullptr),
+ _cpu_time_jfr_locked(UNLOCKED),
+ _has_cpu_time_jfr_requests(false),
+ _cpu_time_jfr_queue(0),
+ _do_async_processing_of_cpu_time_jfr_requests(false)
+#endif
+ {
Thread* thread = Thread::current_or_null();
_parent_trace_id = thread != nullptr ? jvm_thread_id(thread) : (traceid)0;
}
@@ -129,7 +138,9 @@ void JfrThreadLocal::on_start(Thread* t) {
if (JfrRecorder::is_recording()) {
JfrCheckpointManager::write_checkpoint(t);
if (t->is_Java_thread()) {
- send_java_thread_start_event(JavaThread::cast(t));
+ JavaThread *const jt = JavaThread::cast(t);
+ JfrCPUTimeThreadSampling::on_javathread_create(jt);
+ send_java_thread_start_event(jt);
}
}
if (t->jfr_thread_local()->has_cached_stack_trace()) {
@@ -221,6 +232,7 @@ void JfrThreadLocal::on_exit(Thread* t) {
if (t->is_Java_thread()) {
JavaThread* const jt = JavaThread::cast(t);
send_java_thread_end_event(jt, JfrThreadLocal::jvm_thread_id(jt));
+ JfrCPUTimeThreadSampling::on_javathread_terminate(jt);
JfrThreadCPULoadEvent::send_event_for_thread(jt);
}
release(tl, Thread::current()); // because it could be that Thread::current() != t
@@ -537,3 +549,85 @@ Arena* JfrThreadLocal::dcmd_arena(JavaThread* jt) {
tl->_dcmd_arena = arena;
return arena;
}
+
+
+#ifdef LINUX
+
+void JfrThreadLocal::set_cpu_timer(timer_t* timer) {
+ if (_cpu_timer == nullptr) {
+ _cpu_timer = JfrCHeapObj::new_array(1);
+ }
+ *_cpu_timer = *timer;
+}
+
+void JfrThreadLocal::unset_cpu_timer() {
+ if (_cpu_timer != nullptr) {
+ timer_delete(*_cpu_timer);
+ JfrCHeapObj::free(_cpu_timer, sizeof(timer_t));
+ _cpu_timer = nullptr;
+ }
+}
+
+timer_t* JfrThreadLocal::cpu_timer() const {
+ return _cpu_timer;
+}
+
+bool JfrThreadLocal::is_cpu_time_jfr_enqueue_locked() {
+ return Atomic::load_acquire(&_cpu_time_jfr_locked) == ENQUEUE;
+}
+
+bool JfrThreadLocal::is_cpu_time_jfr_dequeue_locked() {
+ return Atomic::load_acquire(&_cpu_time_jfr_locked) == DEQUEUE;
+}
+
+bool JfrThreadLocal::try_acquire_cpu_time_jfr_enqueue_lock() {
+ return Atomic::cmpxchg(&_cpu_time_jfr_locked, UNLOCKED, ENQUEUE) == UNLOCKED;
+}
+
+bool JfrThreadLocal::try_acquire_cpu_time_jfr_dequeue_lock() {
+ CPUTimeLockState got;
+ while (true) {
+ CPUTimeLockState got = Atomic::cmpxchg(&_cpu_time_jfr_locked, UNLOCKED, DEQUEUE);
+ if (got == UNLOCKED) {
+ return true; // successfully locked for dequeue
+ }
+ if (got == DEQUEUE) {
+ return false; // already locked for dequeue
+ }
+ // else wait for the lock to be released from a signal handler
+ }
+}
+
+void JfrThreadLocal::acquire_cpu_time_jfr_dequeue_lock() {
+ while (Atomic::cmpxchg(&_cpu_time_jfr_locked, UNLOCKED, DEQUEUE) != UNLOCKED);
+}
+
+void JfrThreadLocal::release_cpu_time_jfr_queue_lock() {
+ Atomic::release_store(&_cpu_time_jfr_locked, UNLOCKED);
+}
+
+void JfrThreadLocal::set_has_cpu_time_jfr_requests(bool has_requests) {
+ Atomic::release_store(&_has_cpu_time_jfr_requests, has_requests);
+}
+
+bool JfrThreadLocal::has_cpu_time_jfr_requests() {
+ return Atomic::load_acquire(&_has_cpu_time_jfr_requests);
+}
+
+JfrCPUTimeTraceQueue& JfrThreadLocal::cpu_time_jfr_queue() {
+ return _cpu_time_jfr_queue;
+}
+
+void JfrThreadLocal::deallocate_cpu_time_jfr_queue() {
+ cpu_time_jfr_queue().resize(0);
+}
+
+void JfrThreadLocal::set_do_async_processing_of_cpu_time_jfr_requests(bool wants) {
+ Atomic::release_store(&_do_async_processing_of_cpu_time_jfr_requests, wants);
+}
+
+bool JfrThreadLocal::wants_async_processing_of_cpu_time_jfr_requests() {
+ return Atomic::load_acquire(&_do_async_processing_of_cpu_time_jfr_requests);
+}
+
+#endif
diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp
index 8e545d9c429..715a2c44f93 100644
--- a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp
+++ b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp
@@ -33,6 +33,10 @@
#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
+#ifdef LINUX
+#include "jfr/periodic/sampling/jfrCPUTimeThreadSampler.hpp"
+#endif
+
class Arena;
class JavaThread;
class JfrBuffer;
@@ -79,6 +83,22 @@ class JfrThreadLocal {
bool _dead;
bool _sampling_critical_section;
+#ifdef LINUX
+ timer_t* _cpu_timer;
+
+ enum CPUTimeLockState {
+ UNLOCKED,
+ // locked for enqueuing
+ ENQUEUE,
+ // locked for dequeuing
+ DEQUEUE
+ };
+ volatile CPUTimeLockState _cpu_time_jfr_locked;
+ volatile bool _has_cpu_time_jfr_requests;
+ JfrCPUTimeTraceQueue _cpu_time_jfr_queue;
+ volatile bool _do_async_processing_of_cpu_time_jfr_requests;
+#endif
+
JfrBuffer* install_native_buffer() const;
JfrBuffer* install_java_buffer() const;
void release(Thread* t);
@@ -342,6 +362,39 @@ class JfrThreadLocal {
void set_thread_blob(const JfrBlobHandle& handle);
const JfrBlobHandle& thread_blob() const;
+ // CPU time sampling
+#ifdef LINUX
+ void set_cpu_timer(timer_t* timer);
+ void unset_cpu_timer();
+ timer_t* cpu_timer() const;
+
+ // The CPU time JFR lock has three different states:
+ // - ENQUEUE: lock for enqueuing CPU time requests
+ // - DEQUEUE: lock for dequeuing CPU time requests
+ // - UNLOCKED: no lock held
+ // This ensures that we can safely enqueue and dequeue CPU time requests,
+ // without interleaving
+
+ bool is_cpu_time_jfr_enqueue_locked();
+ bool is_cpu_time_jfr_dequeue_locked();
+
+ bool try_acquire_cpu_time_jfr_enqueue_lock();
+ bool try_acquire_cpu_time_jfr_dequeue_lock();
+ void acquire_cpu_time_jfr_dequeue_lock();
+ void release_cpu_time_jfr_queue_lock();
+
+ void set_has_cpu_time_jfr_requests(bool has_events);
+ bool has_cpu_time_jfr_requests();
+
+ JfrCPUTimeTraceQueue& cpu_time_jfr_queue();
+ void deallocate_cpu_time_jfr_queue();
+
+ void set_do_async_processing_of_cpu_time_jfr_requests(bool wants);
+ bool wants_async_processing_of_cpu_time_jfr_requests();
+#else
+ bool has_cpu_time_jfr_requests() { return false; }
+#endif
+
// Hooks
static void on_start(Thread* t);
static void on_exit(Thread* t);
diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
index 2256a0edc99..001a40f74bc 100644
--- a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
+++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
@@ -27,6 +27,7 @@
#include "classfile/symbolTable.hpp"
#include "classfile/systemDictionary.hpp"
#include "classfile/vmClasses.hpp"
+#include "code/nmethod.hpp"
#include "code/scopeDesc.hpp"
#include "compiler/compileBroker.hpp"
#include "compiler/compilerEvent.hpp"
@@ -813,6 +814,14 @@ C2V_VMENTRY_NULL(jobject, lookupConstantInPool, (JNIEnv* env, jobject, ARGUMENT_
return JVMCIENV->get_jobject(JVMCIENV->get_object_constant(obj));
C2V_END
+C2V_VMENTRY_0(jint, getNumIndyEntries, (JNIEnv* env, jobject, ARGUMENT_PAIR(cp)))
+ constantPoolHandle cp(THREAD, UNPACK_PAIR(ConstantPool, cp));
+ if (cp->cache()->resolved_indy_entries() == nullptr) {
+ return 0;
+ }
+ return cp->resolved_indy_entries_length();
+C2V_END
+
C2V_VMENTRY_NULL(jobjectArray, resolveBootstrapMethod, (JNIEnv* env, jobject, ARGUMENT_PAIR(cp), jint index))
constantPoolHandle cp(THREAD, UNPACK_PAIR(ConstantPool, cp));
constantTag tag = cp->tag_at(index);
@@ -1198,7 +1207,7 @@ C2V_VMENTRY_0(jint, installCode0, (JNIEnv *env, jobject,
assert(JVMCIENV->isa_HotSpotNmethod(installed_code_handle), "wrong type");
// Clear the link to an old nmethod first
JVMCIObject nmethod_mirror = installed_code_handle;
- JVMCIENV->invalidate_nmethod_mirror(nmethod_mirror, true, JVMCI_CHECK_0);
+ JVMCIENV->invalidate_nmethod_mirror(nmethod_mirror, true, nmethod::ChangeReason::JVMCI_replacing_with_new_code, JVMCI_CHECK_0);
} else {
assert(JVMCIENV->isa_InstalledCode(installed_code_handle), "wrong type");
}
@@ -1374,7 +1383,7 @@ C2V_VMENTRY(void, reprofile, (JNIEnv* env, jobject, ARGUMENT_PAIR(method)))
nmethod* code = method->code();
if (code != nullptr) {
- code->make_not_entrant("JVMCI reprofile");
+ code->make_not_entrant(nmethod::ChangeReason::JVMCI_reprofile);
}
MethodData* method_data = method->method_data();
@@ -1389,7 +1398,7 @@ C2V_END
C2V_VMENTRY(void, invalidateHotSpotNmethod, (JNIEnv* env, jobject, jobject hs_nmethod, jboolean deoptimize))
JVMCIObject nmethod_mirror = JVMCIENV->wrap(hs_nmethod);
- JVMCIENV->invalidate_nmethod_mirror(nmethod_mirror, deoptimize, JVMCI_CHECK);
+ JVMCIENV->invalidate_nmethod_mirror(nmethod_mirror, deoptimize, nmethod::ChangeReason::JVMCI_invalidate_nmethod, JVMCI_CHECK);
C2V_END
C2V_VMENTRY_NULL(jlongArray, collectCounters, (JNIEnv* env, jobject))
@@ -1814,7 +1823,7 @@ C2V_VMENTRY(void, materializeVirtualObjects, (JNIEnv* env, jobject, jobject _hs_
if (!fst.current()->is_compiled_frame()) {
JVMCI_THROW_MSG(IllegalStateException, "compiled stack frame expected");
}
- fst.current()->cb()->as_nmethod()->make_not_entrant("JVMCI materialize virtual objects");
+ fst.current()->cb()->as_nmethod()->make_not_entrant(nmethod::ChangeReason::JVMCI_materialize_virtual_object);
}
Deoptimization::deoptimize(thread, *fst.current(), Deoptimization::Reason_none);
// look for the frame again as it has been updated by deopt (pc, deopt state...)
@@ -2224,6 +2233,26 @@ C2V_VMENTRY_NULL(jobjectArray, getDeclaredMethods, (JNIEnv* env, jobject, ARGUME
return JVMCIENV->get_jobjectArray(methods);
C2V_END
+C2V_VMENTRY_NULL(jobjectArray, getAllMethods, (JNIEnv* env, jobject, ARGUMENT_PAIR(klass)))
+ Klass* klass = UNPACK_PAIR(Klass, klass);
+ if (klass == nullptr) {
+ JVMCI_THROW_NULL(NullPointerException);
+ }
+ if (!klass->is_instance_klass()) {
+ JVMCIObjectArray methods = JVMCIENV->new_ResolvedJavaMethod_array(0, JVMCI_CHECK_NULL);
+ return JVMCIENV->get_jobjectArray(methods);
+ }
+
+ InstanceKlass* iklass = InstanceKlass::cast(klass);
+ JVMCIObjectArray methods = JVMCIENV->new_ResolvedJavaMethod_array(iklass->methods()->length(), JVMCI_CHECK_NULL);
+ for (int i = 0; i < iklass->methods()->length(); i++) {
+ methodHandle mh(THREAD, iklass->methods()->at(i));
+ JVMCIObject method = JVMCIENV->get_jvmci_method(mh, JVMCI_CHECK_NULL);
+ JVMCIENV->put_object_at(methods, i, method);
+ }
+ return JVMCIENV->get_jobjectArray(methods);
+C2V_END
+
C2V_VMENTRY_NULL(jobjectArray, getDeclaredFieldsInfo, (JNIEnv* env, jobject, ARGUMENT_PAIR(klass)))
Klass* klass = UNPACK_PAIR(Klass, klass);
if (klass == nullptr) {
@@ -3300,6 +3329,7 @@ JNINativeMethod CompilerToVM::methods[] = {
{CC "lookupAppendixInPool", CC "(" HS_CONSTANT_POOL2 "II)" OBJECTCONSTANT, FN_PTR(lookupAppendixInPool)},
{CC "lookupMethodInPool", CC "(" HS_CONSTANT_POOL2 "IB" HS_METHOD2 ")" HS_METHOD, FN_PTR(lookupMethodInPool)},
{CC "lookupConstantInPool", CC "(" HS_CONSTANT_POOL2 "IZ)" JAVACONSTANT, FN_PTR(lookupConstantInPool)},
+ {CC "getNumIndyEntries", CC "(" HS_CONSTANT_POOL2 ")I", FN_PTR(getNumIndyEntries)},
{CC "resolveBootstrapMethod", CC "(" HS_CONSTANT_POOL2 "I)[" OBJECT, FN_PTR(resolveBootstrapMethod)},
{CC "bootstrapArgumentIndexAt", CC "(" HS_CONSTANT_POOL2 "II)I", FN_PTR(bootstrapArgumentIndexAt)},
{CC "getUncachedStringInPool", CC "(" HS_CONSTANT_POOL2 "I)" JAVACONSTANT, FN_PTR(getUncachedStringInPool)},
@@ -3359,6 +3389,7 @@ JNINativeMethod CompilerToVM::methods[] = {
{CC "boxPrimitive", CC "(" OBJECT ")" OBJECTCONSTANT, FN_PTR(boxPrimitive)},
{CC "getDeclaredConstructors", CC "(" HS_KLASS2 ")[" RESOLVED_METHOD, FN_PTR(getDeclaredConstructors)},
{CC "getDeclaredMethods", CC "(" HS_KLASS2 ")[" RESOLVED_METHOD, FN_PTR(getDeclaredMethods)},
+ {CC "getAllMethods", CC "(" HS_KLASS2 ")[" RESOLVED_METHOD, FN_PTR(getAllMethods)},
{CC "getDeclaredFieldsInfo", CC "(" HS_KLASS2 ")[" FIELDINFO, FN_PTR(getDeclaredFieldsInfo)},
{CC "readStaticFieldValue", CC "(" HS_KLASS2 "JC)" JAVACONSTANT, FN_PTR(readStaticFieldValue)},
{CC "readFieldValue", CC "(" OBJECTCONSTANT HS_KLASS2 "JC)" JAVACONSTANT, FN_PTR(readFieldValue)},
diff --git a/src/hotspot/share/jvmci/jvmciEnv.cpp b/src/hotspot/share/jvmci/jvmciEnv.cpp
index b8474552256..851ead247f1 100644
--- a/src/hotspot/share/jvmci/jvmciEnv.cpp
+++ b/src/hotspot/share/jvmci/jvmciEnv.cpp
@@ -1463,8 +1463,7 @@ JVMCIPrimitiveArray JVMCIEnv::new_byteArray(int length, JVMCI_TRAPS) {
JVMCIObjectArray JVMCIEnv::new_byte_array_array(int length, JVMCI_TRAPS) {
JavaThread* THREAD = JavaThread::current(); // For exception macros.
if (is_hotspot()) {
- Klass* byteArrayArrayKlass = TypeArrayKlass::cast(Universe::byteArrayKlass())->array_klass(CHECK_(JVMCIObject()));
- objArrayOop result = ObjArrayKlass::cast(byteArrayArrayKlass) ->allocate(length, CHECK_(JVMCIObject()));
+ objArrayOop result = oopFactory::new_objArray(Universe::byteArrayKlass(), length, CHECK_(JVMCIObject()));
return wrap(result);
} else {
JNIAccessMark jni(this, THREAD);
@@ -1750,7 +1749,7 @@ void JVMCIEnv::initialize_installed_code(JVMCIObject installed_code, CodeBlob* c
}
-void JVMCIEnv::invalidate_nmethod_mirror(JVMCIObject mirror, bool deoptimize, JVMCI_TRAPS) {
+void JVMCIEnv::invalidate_nmethod_mirror(JVMCIObject mirror, bool deoptimize, nmethod::ChangeReason change_reason, JVMCI_TRAPS) {
if (mirror.is_null()) {
JVMCI_THROW(NullPointerException);
}
@@ -1773,7 +1772,7 @@ void JVMCIEnv::invalidate_nmethod_mirror(JVMCIObject mirror, bool deoptimize, JV
if (!deoptimize) {
// Prevent future executions of the nmethod but let current executions complete.
- nm->make_not_entrant("JVMCI invalidate nmethod mirror");
+ nm->make_not_entrant(change_reason);
// Do not clear the address field here as the Java code may still
// want to later call this method with deoptimize == true. That requires
@@ -1782,7 +1781,7 @@ void JVMCIEnv::invalidate_nmethod_mirror(JVMCIObject mirror, bool deoptimize, JV
// Deoptimize the nmethod immediately.
DeoptimizationScope deopt_scope;
deopt_scope.mark(nm);
- nm->make_not_entrant("JVMCI invalidate nmethod mirror");
+ nm->make_not_entrant(change_reason);
nm->make_deoptimized();
deopt_scope.deoptimize_marked();
diff --git a/src/hotspot/share/jvmci/jvmciEnv.hpp b/src/hotspot/share/jvmci/jvmciEnv.hpp
index 69f6647b0d6..b7b7c8f6771 100644
--- a/src/hotspot/share/jvmci/jvmciEnv.hpp
+++ b/src/hotspot/share/jvmci/jvmciEnv.hpp
@@ -462,7 +462,7 @@ public:
// field of `mirror` to prevent it from being called.
// If `deoptimize` is true, the nmethod is immediately deoptimized.
// The HotSpotNmethod.address field is zero upon returning.
- void invalidate_nmethod_mirror(JVMCIObject mirror, bool deoptimze, JVMCI_TRAPS);
+ void invalidate_nmethod_mirror(JVMCIObject mirror, bool deoptimze, nmethod::ChangeReason change_reason, JVMCI_TRAPS);
void initialize_installed_code(JVMCIObject installed_code, CodeBlob* cb, JVMCI_TRAPS);
diff --git a/src/hotspot/share/jvmci/jvmciRuntime.cpp b/src/hotspot/share/jvmci/jvmciRuntime.cpp
index bee5f4ea445..1f10e132eff 100644
--- a/src/hotspot/share/jvmci/jvmciRuntime.cpp
+++ b/src/hotspot/share/jvmci/jvmciRuntime.cpp
@@ -2184,7 +2184,7 @@ JVMCI::CodeInstallResult JVMCIRuntime::register_method(JVMCIEnv* JVMCIENV,
tty->print_cr("Replacing method %s", method_name);
}
if (old != nullptr) {
- old->make_not_entrant("JVMCI register method");
+ old->make_not_entrant(nmethod::ChangeReason::JVMCI_register_method);
}
LogTarget(Info, nmethod, install) lt;
diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
index 4302c0ce6ad..e26c815946d 100644
--- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
+++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
@@ -693,6 +693,7 @@
declare_constant(ConstMethodFlags::_misc_reserved_stack_access) \
declare_constant(ConstMethodFlags::_misc_changes_current_thread) \
declare_constant(ConstMethodFlags::_misc_is_scoped) \
+ declare_constant(ConstMethodFlags::_misc_is_overpass) \
\
declare_constant(CounterData::count_off) \
\
diff --git a/src/hotspot/share/logging/logTag.hpp b/src/hotspot/share/logging/logTag.hpp
index 34927a9b613..9edf81d3f27 100644
--- a/src/hotspot/share/logging/logTag.hpp
+++ b/src/hotspot/share/logging/logTag.hpp
@@ -216,6 +216,7 @@ class outputStream;
LOG_TAG(valuebasedclasses) \
LOG_TAG(verification) \
LOG_TAG(verify) \
+ LOG_TAG(vmatree) \
LOG_TAG(vmmutex) \
LOG_TAG(vmoperation) \
LOG_TAG(vmthread) \
diff --git a/src/hotspot/share/memory/allocation.cpp b/src/hotspot/share/memory/allocation.cpp
index b01ab4014ed..0d99c1bea68 100644
--- a/src/hotspot/share/memory/allocation.cpp
+++ b/src/hotspot/share/memory/allocation.cpp
@@ -86,7 +86,7 @@ void* MetaspaceObj::operator new(size_t size, ClassLoaderData* loader_data,
}
// This is used for allocating training data. We are allocating training data in many cases where a GC cannot be triggered.
-void* MetaspaceObj::operator new(size_t size, MemTag flags) throw() {
+void* MetaspaceObj::operator new(size_t size, MemTag flags) {
void* p = AllocateHeap(size, flags, CALLER_PC);
memset(p, 0, size);
return p;
diff --git a/src/hotspot/share/memory/allocation.hpp b/src/hotspot/share/memory/allocation.hpp
index 8cd6e6d4207..9e500135d0b 100644
--- a/src/hotspot/share/memory/allocation.hpp
+++ b/src/hotspot/share/memory/allocation.hpp
@@ -358,7 +358,7 @@ class MetaspaceObj {
size_t word_size,
Type type) throw();
// This is used for allocating training data. We are allocating training data in many cases where a GC cannot be triggered.
- void* operator new(size_t size, MemTag flags) throw();
+ void* operator new(size_t size, MemTag flags);
void operator delete(void* p) = delete;
// Declare a *static* method with the same signature in any subclass of MetaspaceObj
diff --git a/src/hotspot/share/memory/heap.cpp b/src/hotspot/share/memory/heap.cpp
index bcb9d2e6114..b111c61f15a 100644
--- a/src/hotspot/share/memory/heap.cpp
+++ b/src/hotspot/share/memory/heap.cpp
@@ -189,14 +189,6 @@ static size_t align_to_page_size(size_t size) {
}
-void CodeHeap::on_code_mapping(char* base, size_t size) {
-#ifdef LINUX
- extern void linux_wrap_code(char* base, size_t size);
- linux_wrap_code(base, size);
-#endif
-}
-
-
bool CodeHeap::reserve(ReservedSpace rs, size_t committed_size, size_t segment_size) {
assert(rs.size() >= committed_size, "reserved < committed");
assert(is_aligned(committed_size, rs.page_size()), "must be page aligned");
@@ -213,7 +205,6 @@ bool CodeHeap::reserve(ReservedSpace rs, size_t committed_size, size_t segment_s
return false;
}
- on_code_mapping(_memory.low(), _memory.committed_size());
_number_of_committed_segments = size_to_segments(_memory.committed_size());
_number_of_reserved_segments = size_to_segments(_memory.reserved_size());
assert(_number_of_reserved_segments >= _number_of_committed_segments, "just checking");
@@ -250,7 +241,6 @@ bool CodeHeap::expand_by(size_t size) {
}
char* base = _memory.low() + _memory.committed_size();
if (!_memory.expand_by(dm)) return false;
- on_code_mapping(base, dm);
size_t i = _number_of_committed_segments;
_number_of_committed_segments = size_to_segments(_memory.committed_size());
assert(_number_of_reserved_segments == size_to_segments(_memory.reserved_size()), "number of reserved segments should not change");
diff --git a/src/hotspot/share/memory/heap.hpp b/src/hotspot/share/memory/heap.hpp
index e54f99c8c54..d0e4230fe2b 100644
--- a/src/hotspot/share/memory/heap.hpp
+++ b/src/hotspot/share/memory/heap.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -146,9 +146,6 @@ class CodeHeap : public CHeapObj {
void* next_used(HeapBlock* b) const;
HeapBlock* block_start(void* p) const;
- // to perform additional actions on creation of executable code
- void on_code_mapping(char* base, size_t size);
-
public:
CodeHeap(const char* name, const CodeBlobType code_blob_type);
diff --git a/src/hotspot/share/memory/oopFactory.cpp b/src/hotspot/share/memory/oopFactory.cpp
index 949b4fe134b..b2a468d3a23 100644
--- a/src/hotspot/share/memory/oopFactory.cpp
+++ b/src/hotspot/share/memory/oopFactory.cpp
@@ -40,41 +40,41 @@
#include "utilities/utf8.hpp"
typeArrayOop oopFactory::new_boolArray(int length, TRAPS) {
- return Universe::boolArrayKlass()->allocate(length, THREAD);
+ return Universe::boolArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_charArray(int length, TRAPS) {
- return Universe::charArrayKlass()->allocate(length, THREAD);
+ return Universe::charArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_floatArray(int length, TRAPS) {
- return Universe::floatArrayKlass()->allocate(length, THREAD);
+ return Universe::floatArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_doubleArray(int length, TRAPS) {
- return Universe::doubleArrayKlass()->allocate(length, THREAD);
+ return Universe::doubleArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_byteArray(int length, TRAPS) {
- return Universe::byteArrayKlass()->allocate(length, THREAD);
+ return Universe::byteArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_shortArray(int length, TRAPS) {
- return Universe::shortArrayKlass()->allocate(length, THREAD);
+ return Universe::shortArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_intArray(int length, TRAPS) {
- return Universe::intArrayKlass()->allocate(length, THREAD);
+ return Universe::intArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_longArray(int length, TRAPS) {
- return Universe::longArrayKlass()->allocate(length, THREAD);
+ return Universe::longArrayKlass()->allocate_instance(length, THREAD);
}
// create java.lang.Object[]
objArrayOop oopFactory::new_objectArray(int length, TRAPS) {
assert(Universe::objectArrayKlass() != nullptr, "Too early?");
- return Universe::objectArrayKlass()->allocate(length, THREAD);
+ return Universe::objectArrayKlass()->allocate_instance(length, THREAD);
}
typeArrayOop oopFactory::new_charArray(const char* utf8_str, TRAPS) {
@@ -88,7 +88,7 @@ typeArrayOop oopFactory::new_charArray(const char* utf8_str, TRAPS) {
typeArrayOop oopFactory::new_typeArray(BasicType type, int length, TRAPS) {
TypeArrayKlass* klass = Universe::typeArrayKlass(type);
- return klass->allocate(length, THREAD);
+ return klass->allocate_instance(length, THREAD);
}
// Create a Java array that points to Symbol.
diff --git a/src/hotspot/share/nmt/memoryFileTracker.cpp b/src/hotspot/share/nmt/memoryFileTracker.cpp
index d753a57ede7..4ffcb535e4b 100644
--- a/src/hotspot/share/nmt/memoryFileTracker.cpp
+++ b/src/hotspot/share/nmt/memoryFileTracker.cpp
@@ -91,7 +91,7 @@ void MemoryFileTracker::print_report_on(const MemoryFile* file, outputStream* st
NMTUtil::tag_to_name(prev->val().out.mem_tag()));
{
StreamIndentor si(stream, 4);
- _stack_storage.get(prev->val().out.stack()).print_on(stream);
+ _stack_storage.get(prev->val().out.reserved_stack()).print_on(stream);
}
stream->cr();
}
diff --git a/src/hotspot/share/nmt/nmtNativeCallStackStorage.hpp b/src/hotspot/share/nmt/nmtNativeCallStackStorage.hpp
index 258c3284a18..85e044c1a45 100644
--- a/src/hotspot/share/nmt/nmtNativeCallStackStorage.hpp
+++ b/src/hotspot/share/nmt/nmtNativeCallStackStorage.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -43,11 +43,8 @@
class NativeCallStackStorage : public CHeapObjBase {
public:
using StackIndex = int;
-
-private:
constexpr static const StackIndex invalid = std::numeric_limits::max() - 1;
-public:
static bool equals(const StackIndex a, const StackIndex b) {
return a == b;
}
diff --git a/src/hotspot/share/nmt/vmatree.cpp b/src/hotspot/share/nmt/vmatree.cpp
index 3352a6e5cd4..48febdce7c2 100644
--- a/src/hotspot/share/nmt/vmatree.cpp
+++ b/src/hotspot/share/nmt/vmatree.cpp
@@ -28,21 +28,228 @@
#include "utilities/globalDefinitions.hpp"
#include "utilities/growableArray.hpp"
-const VMATree::RegionData VMATree::empty_regiondata{NativeCallStackStorage::StackIndex{}, mtNone};
+
+// Semantics
+// This tree is used to store and track the state of virtual memory regions.
+// The nodes in the tree are key-value pairs where the key is the memory address and the value is the State of the memory regions.
+// The State of a region describes whether the region is released, reserved or committed, which MemTag it has and where in
+// Hotspot (using call-stacks) it is reserved or committed.
+// Each node holds the State of the regions to its left and right. Each memory region is described by two
+// memory addresses for its start and end.
+// For example, to describe the region that starts at memory address 0xA000 with size 0x1000, there will be two nodes
+// with the keys 0xA000 (node A) and 0xB000 (node B) in the tree. The value of the key-value pairs of node A and
+// node B describe the region's State, using right of A and left of B (<--left--A--right-->.....<--left--B--right-->...).
+//
+// Virtual memory can be reserved, committed, uncommitted and released. For each operation a request
+// () is sent to the tree to handle.
+//
+// The expected changes are described here for each operation:
+//
+// ### Reserve a region
+// When a region is reserved, all the overlapping regions in the tree should:
+// - be marked as Reserved
+// - take MemTag of the operation
+// - store call-stack of the request to the reserve call-stack
+// - clear commit call-stack
+//
+// ### Commit a region
+// When a region is committed, all the overlapping regions in the tree should:
+// - be marked as Committed
+// - take MemTag of the operation or MemTag of the existing region, depends on which-tag-to-use in the request
+// - if the region is in Released state
+// - mark the region as both Reserved and Committed
+// - store the call-stack of the request to the reserve call-stack
+// - store the call-stack of the request to the commit call-stack
+//
+// ### Uncommit a region
+// When a region is uncommitted, all the overlapping regions in the tree should:
+// - be ignored if the region is in Released state
+// - be marked as Reserved
+// - not change the MemTag
+// - not change the reserve call-stack
+// - clear commit call-stack
+//
+// ### Release a region
+// When a region is released, all the overlapping regions in the tree should:
+// - be marked as Released
+// - set the MemTag to mtNone
+// - clear both reserve and commit call-stack
+//
+// --- Accounting
+// After each operation, the tree should be able to report how much memory is reserved or committed per MemTag.
+// So for each region that changes to a new State, the report should contain (separately for each tag) the amount
+// of reserve and commit that are changed (increased or decreased) due to the operation.
+
+const VMATree::RegionData VMATree::empty_regiondata{NativeCallStackStorage::invalid, mtNone};
const char* VMATree::statetype_strings[3] = {
- "reserved", "committed", "released",
+ "released", "reserved", "committed"
};
-VMATree::SummaryDiff VMATree::register_mapping(position A, position B, StateType state,
+VMATree::SIndex VMATree::get_new_reserve_callstack(const SIndex es, const StateType ex, const RequestInfo& req) const {
+ const SIndex ES = NativeCallStackStorage::invalid; // Empty Stack
+ const SIndex rq = req.callstack;
+ const int op = req.op_to_index();
+ const Operation oper = req.op();
+ assert(op >= 0 && op < 4, "should be");
+ assert(op >= 0 && op < 4, "should be");
+ // existing state
+ SIndex result[4][3] = {// Rl Rs C
+ {ES, ES, ES}, // op == Release
+ {rq, rq, rq}, // op == Reserve
+ {es, es, es}, // op == Commit
+ {es, es, es} // op == Uncommit
+ };
+ // When committing a Released region, the reserve-call-stack of the region should also be as what is in the request
+ if (oper == Operation::Commit && ex == StateType::Released) {
+ return rq;
+ } else {
+ return result[op][state_to_index(ex)];
+ }
+}
+
+VMATree::SIndex VMATree::get_new_commit_callstack(const SIndex es, const StateType ex, const RequestInfo& req) const {
+ const SIndex ES = NativeCallStackStorage::invalid; // Empty Stack
+ const SIndex rq = req.callstack;
+ const int op_index = req.op_to_index();
+ const Operation op = req.op();
+ assert(op_index >= 0 && op_index < 4, "should be");
+ // existing state
+ SIndex result[4][3] = {// Rl Rs C
+ {ES, ES, ES}, // op == Release
+ {ES, ES, ES}, // op == Reserve
+ {rq, rq, rq}, // op == Commit
+ {ES, ES, ES} // op == Uncommit
+ };
+ return result[op_index][state_to_index(ex)];
+}
+
+VMATree::StateType VMATree::get_new_state(const StateType ex, const RequestInfo& req) const {
+ const StateType Rl = StateType::Released;
+ const StateType Rs = StateType::Reserved;
+ const StateType C = StateType::Committed;
+ const int op = req.op_to_index();
+ assert(op >= 0 && op < 4, "should be");
+ // existing state
+ StateType result[4][3] = {// Rl Rs C
+ {Rl, Rl, Rl}, // op == Release
+ {Rs, Rs, Rs}, // op == Reserve
+ { C, C, C}, // op == Commit
+ {Rl, Rs, Rs} // op == Uncommit
+ };
+ return result[op][state_to_index(ex)];
+}
+
+MemTag VMATree::get_new_tag(const MemTag ex, const RequestInfo& req) const {
+ switch(req.op()) {
+ case Operation::Release:
+ return mtNone;
+ case Operation::Reserve:
+ return req.tag;
+ case Operation::Commit:
+ return req.use_tag_inplace ? ex : req.tag;
+ case Operation::Uncommit:
+ return ex;
+ default:
+ break;
+ }
+ return mtNone;
+}
+
+void VMATree::compute_summary_diff(const SingleDiff::delta region_size,
+ const MemTag current_tag,
+ const StateType& ex,
+ const RequestInfo& req,
+ const MemTag operation_tag,
+ SummaryDiff& diff) const {
+ const StateType Rl = StateType::Released;
+ const StateType Rs = StateType::Reserved;
+ const StateType C = StateType::Committed;
+ const int op = req.op_to_index();
+ const Operation oper = req.op();
+ assert(op >= 0 && op < 4, "should be");
+
+ SingleDiff::delta a = region_size;
+ // A region with size `a` has a state as and an operation is requested as in
+ // The region has tag `current_tag` and the operation has tag `operation_tag`.
+ // For each state, we decide how much to be added/subtracted from current_tag to operation_tag. Two tables for reserve and commit.
+ // Each pair of in the table means add `x` to current_tag and add `y` to operation_tag. There are 3 pairs in each row for 3 states.
+ // For example, `reserve[1][4,5]` says `-a,a` means:
+ // - we are reserving with operation_tag a region which is already commited with current_tag
+ // - since we are reserving, then `a` will be added to operation_tag. (`y` is `a`)
+ // - since we uncommitting (by reserving) then `a` is to be subtracted from current_tag. (`x` is `-a`).
+ // - amount of uncommitted size is in table `commit[1][4,5]` which is `-a,0` that means subtract `a` from current_tag.
+ // existing state
+ SingleDiff::delta reserve[4][3*2] = {// Rl Rs C
+ {0,0, -a,0, -a,0 }, // op == Release
+ {0,a, -a,a, -a,a }, // op == Reserve
+ {0,a, -a,a, -a,a }, // op == Commit
+ {0,0, 0,0, 0,0 } // op == Uncommit
+ };
+ SingleDiff::delta commit[4][3*2] = {// Rl Rs C
+ {0,0, 0,0, -a,0 }, // op == Release
+ {0,0, 0,0, -a,0 }, // op == Reserve
+ {0,a, 0,a, -a,a }, // op == Commit
+ {0,0, 0,0, -a,0 } // op == Uncommit
+ };
+ SingleDiff& from_rescom = diff.tag[NMTUtil::tag_to_index(current_tag)];
+ SingleDiff& to_rescom = diff.tag[NMTUtil::tag_to_index(operation_tag)];
+ int st = state_to_index(ex);
+ from_rescom.reserve += reserve[op][st * 2 ];
+ to_rescom.reserve += reserve[op][st * 2 + 1];
+ from_rescom.commit += commit[op][st * 2 ];
+ to_rescom.commit += commit[op][st * 2 + 1];
+
+}
+// update the region state between n1 and n2. Since n1 and n2 are pointers, any update of them will be visible from tree.
+// If n1 is noop, it can be removed because its left region (n1->val().in) is already decided and its right state (n1->val().out) is decided here.
+// The state of right of n2 (n2->val().out) cannot be decided here yet.
+void VMATree::update_region(TreapNode* n1, TreapNode* n2, const RequestInfo& req, SummaryDiff& diff) {
+ assert(n1 != nullptr,"sanity");
+ assert(n2 != nullptr,"sanity");
+ //.........n1......n2......
+ // ^------^
+ // |
+ IntervalState exSt = n1->val().out; // existing state info
+
+
+ StateType existing_state = exSt.type();
+ MemTag existing_tag = exSt.mem_tag();
+ SIndex existing_reserve_callstack = exSt.reserved_stack();
+ SIndex existing_commit_callstack = exSt.committed_stack();
+
+ StateType new_state = get_new_state(existing_state, req);
+ MemTag new_tag = get_new_tag(n1->val().out.mem_tag(), req);
+ SIndex new_reserve_callstack = get_new_reserve_callstack(existing_reserve_callstack, existing_state, req);
+ SIndex new_commit_callstack = get_new_commit_callstack(existing_commit_callstack, existing_state, req);
+
+ // n1........n2
+ // out-->
+ n1->val().out.set_tag(new_tag);
+ n1->val().out.set_type(new_state);
+ n1->val().out.set_reserve_stack(new_reserve_callstack);
+ n1->val().out.set_commit_stack(new_commit_callstack);
+
+ // n1........n2
+ // <--in
+ n2->val().in.set_tag(new_tag);
+ n2->val().in.set_type(new_state);
+ n2->val().in.set_reserve_stack(new_reserve_callstack);
+ n2->val().in.set_commit_stack(new_commit_callstack);
+
+ SingleDiff::delta region_size = n2->key() - n1->key();
+ compute_summary_diff(region_size, existing_tag, existing_state, req, new_tag, diff);
+}
+
+VMATree::SummaryDiff VMATree::register_mapping(position _A, position _B, StateType state,
const RegionData& metadata, bool use_tag_inplace) {
- assert(!use_tag_inplace || metadata.mem_tag == mtNone,
- "If using use_tag_inplace, then the supplied tag should be mtNone, was instead: %s", NMTUtil::tag_to_name(metadata.mem_tag));
- if (A == B) {
- // A 0-sized mapping isn't worth recording.
+
+ if (_A == _B) {
return SummaryDiff();
}
-
+ assert(_A < _B, "should be");
+ SummaryDiff diff;
+ RequestInfo req{_A, _B, state, metadata.mem_tag, metadata.stack_idx, use_tag_inplace};
IntervalChange stA{
IntervalState{StateType::Released, empty_regiondata},
IntervalState{ state, metadata}
@@ -51,176 +258,400 @@ VMATree::SummaryDiff VMATree::register_mapping(position A, position B, StateType
IntervalState{ state, metadata},
IntervalState{StateType::Released, empty_regiondata}
};
+ stA.out.set_commit_stack(NativeCallStackStorage::invalid);
+ stB.in.set_commit_stack(NativeCallStackStorage::invalid);
+ VMATreap::Range rA = _tree.find_enclosing_range(_A);
+ VMATreap::Range rB = _tree.find_enclosing_range(_B);
- // First handle A.
- // Find closest node that is LEQ A
- bool LEQ_A_found = false;
- AddressState LEQ_A;
- TreapNode* leqA_n = _tree.closest_leq(A);
- if (leqA_n == nullptr) {
- assert(!use_tag_inplace, "Cannot use the tag inplace if no pre-existing tag exists. From: " PTR_FORMAT " To: " PTR_FORMAT, A, B);
- if (use_tag_inplace) {
- log_debug(nmt)("Cannot use the tag inplace if no pre-existing tag exists. From: " PTR_FORMAT " To: " PTR_FORMAT, A, B);
+ // nodes: .....X.......Y...Z......W........U
+ // request: A------------------B
+ // X,Y = enclosing_nodes(A)
+ // W,U = enclosing_nodes(B)
+ // The cases are whether or not X and Y exists and X == A. (A == Y doesn't happen since it is searched by 'lt' predicate)
+ // The cases are whether or not W and U exists and W == B. (B == U doesn't happen since it is searched by 'lt' predicate)
+
+ // We update regions in 3 sections: 1) X..A..Y, 2) Y....W, 3) W..B..U
+ // Y: is the closest node greater than A, but less than B
+ // W: is the closest node less than B, but greater than A
+ // The regions in [Y,W) are updated in a loop. We update X..A..Y before the loop and W..B..U after the loop.
+ // The table below summarizes the overlap cases. The overlapping case depends on whether X, Y, W and U exist or not,
+ // and if they exist whether they are the same or not.
+ // In the notations here, when there is not dot ('.') between two nodes it meaans that they are the same. For example,
+ // ...XA....Y.... means X == A.
+
+
+ // row 0: .........A..................B.....
+ // row 1: .........A...YW.............B..... // it is impossible, since it means only one node exists in the tree.
+ // row 2: .........A...Y..........W...B.....
+ // row 3: .........A...Y.............WB.....
+
+ // row 4: .....X...A..................B.....
+ // row 5: .....X...A...YW.............B.....
+ // row 6: .....X...A...Y..........W...B.....
+ // row 7: .....X...A...Y.............WB.....
+
+ // row 8: ........XA..................B.....
+ // row 9: ........XA...YW.............B.....
+ // row 10: ........XA...Y..........W...B.....
+ // row 11: ........XA...Y.............WB.....
+
+ // row 12: .........A..................B....U
+ // row 13: .........A...YW.............B....U
+ // row 14: .........A...Y..........W...B....U
+ // row 15: .........A...Y.............WB....U
+
+ // row 16: .....X...A..................B....U
+ // row 17: .....X...A...YW.............B....U
+ // row 18: .....X...A...Y..........W...B....U
+ // row 19: .....X...A...Y.............WB....U
+
+ // row 20: ........XA..................B....U
+ // row 21: ........XA...YW.............B....U
+ // row 22: ........XA...Y..........W...B....U
+ // row 23: ........XA...Y.............WB....U
+
+
+ // We intentionally did not summarize/compress the cases to keep them as separate.
+ // This expanded way of describing the cases helps us to understand/analyze/verify/debug/maintain
+ // the corresponding code more easily.
+ // Mapping of table to row, row to switch-case should be consistent. If one changes, the others have
+ // to be updated accordingly. The sequence of dependecies is: table -> row no -> switch(row)-case -> code.
+ // Meaning that whenever any of one item in this sequence is changed, the rest of the consequent items to
+ // be checked/changed.
+
+ TreapNode* X = rA.start;
+ TreapNode* Y = rA.end;
+ TreapNode* W = rB.start;
+ TreapNode* U = rB.end;
+ TreapNode nA{_A, stA, 0}; // the node that represents A
+ TreapNode nB{_B, stB, 0}; // the node that represents B
+ TreapNode* A = &nA;
+ TreapNode* B = &nB;
+ auto upsert_if= [&](TreapNode* node) {
+ if (!node->val().is_noop()) {
+ _tree.upsert(node->key(), node->val());
}
- // No match. We add the A node directly, unless it would have no effect.
- if (!stA.is_noop()) {
- _tree.upsert(A, stA);
+ };
+ // update region between n1 and n2
+ auto update = [&](TreapNode* n1, TreapNode* n2) {
+ update_region(n1, n2, req, diff);
+ };
+ auto remove_if = [&](TreapNode* node) -> bool{
+ if (node->val().is_noop()) {
+ _tree.remove(node->key());
+ return true;
}
- } else {
- LEQ_A_found = true;
- LEQ_A = AddressState{leqA_n->key(), leqA_n->val()};
- StateType leqA_state = leqA_n->val().out.type();
- StateType new_state = stA.out.type();
- // If we specify use_tag_inplace then the new region takes over the current tag instead of the tag in metadata.
- // This is important because the VirtualMemoryTracker API doesn't require supplying the tag for some operations.
- if (use_tag_inplace) {
- assert(leqA_n->val().out.type() != StateType::Released, "Should not use inplace the tag of a released region");
- MemTag tag = leqA_n->val().out.mem_tag();
- stA.out.set_tag(tag);
- stB.in.set_tag(tag);
- }
-
- // Unless we know better, let B's outgoing state be the outgoing state of the node at or preceding A.
- // Consider the case where the found node is the start of a region enclosing [A,B)
- stB.out = out_state(leqA_n);
-
- // Direct address match.
- if (leqA_n->key() == A) {
- // Take over in state from old address.
- stA.in = in_state(leqA_n);
-
- // We may now be able to merge two regions:
- // If the node's old state matches the new, it becomes a noop. That happens, for example,
- // when expanding a committed area: commit [x1, A); ... commit [A, x3)
- // and the result should be a larger area, [x1, x3). In that case, the middle node (A and le_n)
- // is not needed anymore. So we just remove the old node.
- stB.in = stA.out;
- if (stA.is_noop()) {
- // invalidates leqA_n
- _tree.remove(leqA_n->key());
- } else {
- // If the state is not matching then we have different operations, such as:
- // reserve [x1, A); ... commit [A, x2); or
- // reserve [x1, A), mem_tag1; ... reserve [A, x2), mem_tag2; or
- // reserve [A, x1), mem_tag1; ... reserve [A, x2), mem_tag2;
- // then we re-use the existing out node, overwriting its old metadata.
- leqA_n->val() = stA;
+ return false;
+ };
+ GrowableArrayCHeap to_be_removed;
+ // update regions in [Y,W)
+ auto update_loop = [&]() {
+ TreapNode* prev = nullptr;
+ _tree.visit_range_in_order(_A + 1, _B + 1, [&](TreapNode* curr) {
+ if (prev != nullptr) {
+ update_region(prev, curr, req, diff);
+ // during visit, structure of the tree should not be changed
+ // keep the keys to be removed, and remove them later
+ if (prev->val().is_noop()) {
+ to_be_removed.push(prev->key());
+ }
}
- } else {
- // The address must be smaller.
- assert(A > leqA_n->key(), "must be");
+ prev = curr;
+ });
+ };
+ // update region of [A,T)
+ auto update_A = [&](TreapNode* T) {
+ A->val().out = A->val().in;
+ update(A, T);
+ };
+ bool X_exists = X != nullptr;
+ bool Y_exists = Y != nullptr && Y->key() <= _B;
+ bool W_exists = W != nullptr && W->key() > _A;
+ bool U_exists = U != nullptr;
+ bool X_eq_A = X_exists && X->key() == _A;
+ bool W_eq_B = W_exists && W->key() == _B;
+ bool Y_eq_W = Y_exists && W_exists && W->key() == Y->key();
+ int row = -1;
+#ifdef ASSERT
+ auto print_case = [&]() {
+ log_trace(vmatree)(" req: %4d---%4d", (int)_A, (int)_B);
+ log_trace(vmatree)(" row: %2d", row);
+ log_trace(vmatree)(" X: %4ld", X_exists ? (long)X->key() : -1);
+ log_trace(vmatree)(" Y: %4ld", Y_exists ? (long)Y->key() : -1);
+ log_trace(vmatree)(" W: %4ld", W_exists ? (long)W->key() : -1);
+ log_trace(vmatree)(" U: %4ld", U_exists ? (long)U->key() : -1);
+ };
+#endif
+ // Order of the nodes if they exist are as: X <= A < Y <= W <= B < U
+ // A---------------------------B
+ // X Y YW WB U
+ // XA Y YW WB U
+ if (!X_exists && !Y_exists && !U_exists) { row = 0; }
+ if (!X_exists && Y_exists && Y_eq_W && !W_eq_B && !U_exists) { row = 1; }
+ if (!X_exists && Y_exists && !Y_eq_W && !W_eq_B && !U_exists) { row = 2; }
+ if (!X_exists && Y_exists && W_eq_B && !U_exists) { row = 3; }
- // We add a new node, but only if there would be a state change. If there would not be a
- // state change, we just omit the node.
- // That happens, for example, when reserving within an already reserved region with identical metadata.
- stA.in = out_state(leqA_n); // .. and the region's prior state is the incoming state
- if (stA.is_noop()) {
- // Nothing to do.
- } else {
- // Add new node.
- _tree.upsert(A, stA);
- }
+ if ( X_exists && !Y_exists && !U_exists) { row = 4; }
+ if ( X_exists && Y_exists && Y_eq_W && !W_eq_B && !U_exists) { row = 5; }
+ if ( X_exists && Y_exists && !Y_eq_W && !W_eq_B && !U_exists) { row = 6; }
+ if ( X_exists && Y_exists && W_eq_B && !U_exists) { row = 7; }
+
+ if ( X_eq_A && !Y_exists && !U_exists) { row = 8; }
+ if ( X_eq_A && Y_exists && Y_eq_W && !W_eq_B && !U_exists) { row = 9; }
+ if ( X_eq_A && Y_exists && !Y_eq_W && !W_eq_B && !U_exists) { row = 10; }
+ if ( X_eq_A && Y_exists && W_eq_B && !U_exists) { row = 11; }
+
+ if (!X_exists && !Y_exists && U_exists) { row = 12; }
+ if (!X_exists && Y_exists && Y_eq_W && !W_eq_B && U_exists) { row = 13; }
+ if (!X_exists && Y_exists && !Y_eq_W && !W_eq_B && U_exists) { row = 14; }
+ if (!X_exists && Y_exists && W_eq_B && U_exists) { row = 15; }
+
+ if ( X_exists && !Y_exists && U_exists) { row = 16; }
+ if ( X_exists && Y_exists && Y_eq_W && !W_eq_B && U_exists) { row = 17; }
+ if ( X_exists && Y_exists && !Y_eq_W && !W_eq_B && U_exists) { row = 18; }
+ if ( X_exists && Y_exists && W_eq_B && U_exists) { row = 19; }
+
+ if ( X_eq_A && !Y_exists && U_exists) { row = 20; }
+ if ( X_eq_A && Y_exists && Y_eq_W && !W_eq_B && U_exists) { row = 21; }
+ if ( X_eq_A && Y_exists && !Y_eq_W && !W_eq_B && U_exists) { row = 22; }
+ if ( X_eq_A && Y_exists && W_eq_B && U_exists) { row = 23; }
+
+ DEBUG_ONLY(print_case();)
+ switch(row) {
+ // row 0: .........A..................B.....
+ case 0: {
+ update_A(B);
+ upsert_if(A);
+ upsert_if(B);
+ break;
}
- }
-
- // Now we handle B.
- // We first search all nodes that are (A, B]. All of these nodes
- // need to be deleted and summary accounted for. The last node before B determines B's outgoing state.
- // If there is no node between A and B, its A's incoming state.
- GrowableArrayCHeap to_be_deleted_inbetween_a_b;
- bool B_needs_insert = true;
-
- // Find all nodes between (A, B] and record their addresses and values. Also update B's
- // outgoing state.
- _tree.visit_range_in_order(A + 1, B + 1, [&](TreapNode* head) {
- int cmp_B = PositionComparator::cmp(head->key(), B);
- stB.out = out_state(head);
- if (cmp_B < 0) {
- // Record all nodes preceding B.
- to_be_deleted_inbetween_a_b.push({head->key(), head->val()});
- } else if (cmp_B == 0) {
- // Re-purpose B node, unless it would result in a noop node, in
- // which case record old node at B for deletion and summary accounting.
- if (stB.is_noop()) {
- to_be_deleted_inbetween_a_b.push(AddressState{B, head->val()});
- } else {
- head->val() = stB;
- }
- B_needs_insert = false;
+ // row 1: .........A...YW.............B.....
+ case 1: {
+ ShouldNotReachHere();
+ break;
}
- });
-
- // Insert B node if needed
- if (B_needs_insert && // Was not already inserted
- !stB.is_noop()) // The operation is differing
- {
- _tree.upsert(B, stB);
- }
-
- // We now need to:
- // a) Delete all nodes between (A, B]. Including B in the case of a noop.
- // b) Perform summary accounting
- SummaryDiff diff;
-
- if (to_be_deleted_inbetween_a_b.length() == 0 && LEQ_A_found) {
- // We must have smashed a hole in an existing region (or replaced it entirely).
- // LEQ_A < A < B <= C
- SingleDiff& rescom = diff.tag[NMTUtil::tag_to_index(LEQ_A.out().mem_tag())];
- if (LEQ_A.out().type() == StateType::Reserved) {
- rescom.reserve -= B - A;
- } else if (LEQ_A.out().type() == StateType::Committed) {
- rescom.commit -= B - A;
- rescom.reserve -= B - A;
+ // row 2: .........A...Y..........W...B.....
+ case 2: {
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ remove_if(Y);
+ update(W, B);
+ remove_if(W);
+ upsert_if(B);
+ break;
}
- }
-
- // Track the previous node.
- AddressState prev{A, stA};
- for (int i = 0; i < to_be_deleted_inbetween_a_b.length(); i++) {
- const AddressState delete_me = to_be_deleted_inbetween_a_b.at(i);
- _tree.remove(delete_me.address);
-
- // Perform summary accounting
- SingleDiff& rescom = diff.tag[NMTUtil::tag_to_index(delete_me.in().mem_tag())];
- if (delete_me.in().type() == StateType::Reserved) {
- rescom.reserve -= delete_me.address - prev.address;
- } else if (delete_me.in().type() == StateType::Committed) {
- rescom.commit -= delete_me.address - prev.address;
- rescom.reserve -= delete_me.address - prev.address;
+ // row 3: .........A...Y.............WB.....
+ case 3: {
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ remove_if(W);
+ break;
}
- prev = delete_me;
- }
-
- if (prev.address != A && prev.out().type() != StateType::Released) {
- // The last node wasn't released, so it must be connected to a node outside of (A, B)
- // A - prev - B - (some node >= B)
- // It might be that prev.address == B == (some node >= B), this is fine.
- if (prev.out().type() == StateType::Reserved) {
- SingleDiff& rescom = diff.tag[NMTUtil::tag_to_index(prev.out().mem_tag())];
- rescom.reserve -= B - prev.address;
- } else if (prev.out().type() == StateType::Committed) {
- SingleDiff& rescom = diff.tag[NMTUtil::tag_to_index(prev.out().mem_tag())];
- rescom.commit -= B - prev.address;
- rescom.reserve -= B - prev.address;
+ // row 4: .....X...A..................B.....
+ case 4: {
+ A->val().in = X->val().out;
+ update_A(B);
+ upsert_if(A);
+ upsert_if(B);
+ break;
}
+ // row 5: .....X...A...YW.............B.....
+ case 5: {
+ A->val().in = X->val().out;
+ update_A(Y);
+ upsert_if(A);
+ update(Y, B);
+ remove_if(Y);
+ upsert_if(B);
+ break;
+ }
+ // row 6: .....X...A...Y..........W...B.....
+ case 6: {
+ A->val().in = X->val().out;
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ update(W, B);
+ remove_if(W);
+ upsert_if(B);
+ break;
+ }
+ // row 7: .....X...A...Y.............WB.....
+ case 7: {
+ A->val().in = X->val().out;
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ remove_if(W);
+ break;
+ }
+ // row 8: ........XA..................B.....
+ case 8: {
+ update(X, B);
+ remove_if(X);
+ upsert_if(B);
+ break;
+ }
+ // row 9: ........XA...YW.............B.....
+ case 9: {
+ update(X, Y);
+ remove_if(X);
+ update(W, B);
+ remove_if(W);
+ upsert_if(B);
+ break;
+ }
+ // row 10: ........XA...Y..........W...B.....
+ case 10: {
+ update(X, Y);
+ remove_if(X);
+ update_loop();
+ update(W, B);
+ remove_if(W);
+ upsert_if(B);
+ break;
+ }
+ // row 11: ........XA...Y.............WB.....
+ case 11: {
+ update(X, Y);
+ remove_if(X);
+ update_loop();
+ remove_if(W);
+ break;
+ }
+ // row 12: .........A..................B....U
+ case 12: {
+ update_A(B);
+ upsert_if(A);
+ upsert_if(B);
+ break;
+ }
+ // row 13: .........A...YW.............B....U
+ case 13: {
+ update_A(Y);
+ upsert_if(A);
+ update(W, B);
+ remove_if(W);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 14: .........A...Y..........W...B....U
+ case 14: {
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ update(W, B);
+ remove_if(W);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 15: .........A...Y.............WB....U
+ case 15: {
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ remove_if(W);
+ break;
+ }
+ // row 16: .....X...A..................B....U
+ case 16: {
+ A->val().in = X->val().out;
+ update_A(B);
+ upsert_if(A);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 17: .....X...A...YW.............B....U
+ case 17: {
+ A->val().in = X->val().out;
+ update_A(Y);
+ upsert_if(A);
+ update(W, B);
+ remove_if(W);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 18: .....X...A...Y..........W...B....U
+ case 18: {
+ A->val().in = X->val().out;
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ update(W, B);
+ remove_if(W);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 19: .....X...A...Y.............WB....U
+ case 19: {
+ A->val().in = X->val().out;
+ update_A(Y);
+ upsert_if(A);
+ update_loop();
+ remove_if(W);
+ break;
+ }
+ // row 20: ........XA..................B....U
+ case 20: {
+ update(X, B);
+ remove_if(X);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 21: ........XA...YW.............B....U
+ case 21: {
+ update(X, Y);
+ remove_if(X);
+ update(W, B);
+ remove_if(W);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 22: ........XA...Y..........W...B....U
+ case 22: {
+ update(X, Y);
+ remove_if(X);
+ update_loop();
+ update(W, B);
+ remove_if(W);
+ B->val().out = U->val().in;
+ upsert_if(B);
+ break;
+ }
+ // row 23: ........XA...Y.............WB....U
+ case 23: {
+ update(X, Y);
+ remove_if(X);
+ update_loop();
+ remove_if(W);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
}
- // Finally, we can register the new region [A, B)'s summary data.
- SingleDiff& rescom = diff.tag[NMTUtil::tag_to_index(stA.out.mem_tag())];
- if (state == StateType::Reserved) {
- rescom.reserve += B - A;
- } else if (state == StateType::Committed) {
- rescom.commit += B - A;
- rescom.reserve += B - A;
+ // Remove the 'noop' nodes that found inside the loop
+ while(to_be_removed.length() != 0) {
+ _tree.remove(to_be_removed.pop());
}
+
return diff;
}
#ifdef ASSERT
void VMATree::print_on(outputStream* out) {
visit_in_order([&](TreapNode* current) {
- out->print("%zu (%s) - %s - ", current->key(), NMTUtil::tag_to_name(out_state(current).mem_tag()),
- statetype_to_string(out_state(current).type()));
+ out->print("%zu (%s) - %s [%d, %d]-> ", current->key(), NMTUtil::tag_to_name(out_state(current).mem_tag()),
+ statetype_to_string(out_state(current).type()), current->val().out.reserved_stack(), current->val().out.committed_stack());
});
out->cr();
}
@@ -268,7 +699,7 @@ VMATree::SummaryDiff VMATree::set_tag(const position start, const size size, con
SummaryDiff diff;
// Ignore any released ranges, these must be mtNone and have no stack
if (type != StateType::Released) {
- RegionData new_data = RegionData(out.stack(), tag);
+ RegionData new_data = RegionData(out.reserved_stack(), tag);
SummaryDiff result = register_mapping(from, end, type, new_data);
diff.add(result);
}
@@ -289,7 +720,7 @@ VMATree::SummaryDiff VMATree::set_tag(const position start, const size size, con
StateType type = out.type();
if (type != StateType::Released) {
- RegionData new_data = RegionData(out.stack(), tag);
+ RegionData new_data = RegionData(out.reserved_stack(), tag);
SummaryDiff result = register_mapping(from, end, type, new_data);
diff.add(result);
}
diff --git a/src/hotspot/share/nmt/vmatree.hpp b/src/hotspot/share/nmt/vmatree.hpp
index 0c639e929b7..d3bf3724c0e 100644
--- a/src/hotspot/share/nmt/vmatree.hpp
+++ b/src/hotspot/share/nmt/vmatree.hpp
@@ -44,6 +44,7 @@ class VMATree {
public:
using position = size_t;
using size = size_t;
+ using SIndex = NativeCallStackStorage::StackIndex;
class PositionComparator {
public:
@@ -55,7 +56,7 @@ public:
}
};
- enum class StateType : uint8_t { Reserved, Committed, Released, LAST };
+ enum class StateType : uint8_t { Released, Reserved, Committed, LAST };
private:
static const char* statetype_strings[static_cast(StateType::LAST)];
@@ -70,12 +71,12 @@ public:
// Each point has some stack and a tag associated with it.
struct RegionData {
- const NativeCallStackStorage::StackIndex stack_idx;
+ const SIndex stack_idx;
const MemTag mem_tag;
RegionData() : stack_idx(), mem_tag(mtNone) {}
- RegionData(NativeCallStackStorage::StackIndex stack_idx, MemTag mem_tag)
+ RegionData(SIndex stack_idx, MemTag mem_tag)
: stack_idx(stack_idx), mem_tag(mem_tag) {}
static bool equals(const RegionData& a, const RegionData& b) {
@@ -91,15 +92,27 @@ private:
private:
// Store the type and mem_tag as two bytes
uint8_t type_tag[2];
- NativeCallStackStorage::StackIndex sidx;
+ NativeCallStackStorage::StackIndex _reserved_stack;
+ NativeCallStackStorage::StackIndex _committed_stack;
public:
- IntervalState() : type_tag{0,0}, sidx() {}
+ IntervalState() : type_tag{0,0}, _reserved_stack(NativeCallStackStorage::invalid), _committed_stack(NativeCallStackStorage::invalid) {}
+ IntervalState(const StateType type,
+ const MemTag mt,
+ const NativeCallStackStorage::StackIndex res_stack,
+ const NativeCallStackStorage::StackIndex com_stack) {
+ assert(!(type == StateType::Released) || mt == mtNone, "Released state-type must have memory tag mtNone");
+ type_tag[0] = static_cast(type);
+ type_tag[1] = static_cast(mt);
+ _reserved_stack = res_stack;
+ _committed_stack = com_stack;
+ }
IntervalState(const StateType type, const RegionData data) {
assert(!(type == StateType::Released) || data.mem_tag == mtNone, "Released state-type must have memory tag mtNone");
type_tag[0] = static_cast(type);
type_tag[1] = static_cast(data.mem_tag);
- sidx = data.stack_idx;
+ _reserved_stack = data.stack_idx;
+ _committed_stack = NativeCallStackStorage::invalid;
}
StateType type() const {
@@ -110,16 +123,50 @@ private:
return static_cast(type_tag[1]);
}
- RegionData regiondata() const {
- return RegionData{sidx, mem_tag()};
+ RegionData reserved_regiondata() const {
+ return RegionData{_reserved_stack, mem_tag()};
+ }
+ RegionData committed_regiondata() const {
+ return RegionData{_committed_stack, mem_tag()};
}
void set_tag(MemTag tag) {
type_tag[1] = static_cast(tag);
}
- NativeCallStackStorage::StackIndex stack() const {
- return sidx;
+ NativeCallStackStorage::StackIndex reserved_stack() const {
+ return _reserved_stack;
+ }
+
+ NativeCallStackStorage::StackIndex committed_stack() const {
+ return _committed_stack;
+ }
+
+ void set_reserve_stack(NativeCallStackStorage::StackIndex idx) {
+ _reserved_stack = idx;
+ }
+
+ void set_commit_stack(NativeCallStackStorage::StackIndex idx) {
+ _committed_stack = idx;
+ }
+
+ bool has_reserved_stack() {
+ return _reserved_stack != NativeCallStackStorage::invalid;
+ }
+
+ bool has_committed_stack() {
+ return _committed_stack != NativeCallStackStorage::invalid;
+ }
+
+ void set_type(StateType t) {
+ type_tag[0] = static_cast(t);
+ }
+
+ bool equals(const IntervalState& other) const {
+ return mem_tag() == other.mem_tag() &&
+ type() == other.type() &&
+ reserved_stack() == other.reserved_stack() &&
+ committed_stack() == other.committed_stack();
}
};
@@ -130,8 +177,14 @@ private:
IntervalState out;
bool is_noop() {
+ if (in.type() == StateType::Released &&
+ in.type() == out.type() &&
+ in.mem_tag() == out.mem_tag()) {
+ return true;
+ }
return in.type() == out.type() &&
- RegionData::equals(in.regiondata(), out.regiondata());
+ RegionData::equals(in.reserved_regiondata(), out.reserved_regiondata()) &&
+ RegionData::equals(in.committed_regiondata(), out.committed_regiondata());
}
};
@@ -193,8 +246,44 @@ public:
#endif
};
+ enum Operation {Release, Reserve, Commit, Uncommit};
+ struct RequestInfo {
+ position A, B;
+ StateType _op;
+ MemTag tag;
+ SIndex callstack;
+ bool use_tag_inplace;
+ Operation op() const {
+ return
+ _op == StateType::Reserved && !use_tag_inplace ? Operation::Reserve :
+ _op == StateType::Committed ? Operation::Commit :
+ _op == StateType::Reserved && use_tag_inplace ? Operation::Uncommit :
+ Operation::Release;
+ }
+
+ int op_to_index() const {
+ return
+ _op == StateType::Reserved && !use_tag_inplace ? 1 :
+ _op == StateType::Committed ? 2 :
+ _op == StateType::Reserved && use_tag_inplace ? 3 :
+ 0;
+ }
+ };
+
private:
SummaryDiff register_mapping(position A, position B, StateType state, const RegionData& metadata, bool use_tag_inplace = false);
+ StateType get_new_state(const StateType existinting_state, const RequestInfo& req) const;
+ MemTag get_new_tag(const MemTag existinting_tag, const RequestInfo& req) const;
+ SIndex get_new_reserve_callstack(const SIndex existinting_stack, const StateType ex, const RequestInfo& req) const;
+ SIndex get_new_commit_callstack(const SIndex existinting_stack, const StateType ex, const RequestInfo& req) const;
+ void compute_summary_diff(const SingleDiff::delta region_size, const MemTag t1, const StateType& ex, const RequestInfo& req, const MemTag new_tag, SummaryDiff& diff) const;
+ void update_region(TreapNode* n1, TreapNode* n2, const RequestInfo& req, SummaryDiff& diff);
+ int state_to_index(const StateType st) const {
+ return
+ st == StateType::Released ? 0 :
+ st == StateType::Reserved ? 1 :
+ st == StateType::Committed ? 2 : -1;
+ }
public:
SummaryDiff reserve_mapping(position from, size size, const RegionData& metadata) {
diff --git a/src/hotspot/share/oops/fieldInfo.cpp b/src/hotspot/share/oops/fieldInfo.cpp
index 300b45277ad..d0825ba6df8 100644
--- a/src/hotspot/share/oops/fieldInfo.cpp
+++ b/src/hotspot/share/oops/fieldInfo.cpp
@@ -22,8 +22,11 @@
*
*/
+#include "memory/resourceArea.hpp"
+#include "cds/cdsConfig.hpp"
#include "oops/fieldInfo.inline.hpp"
#include "runtime/atomic.hpp"
+#include "utilities/packedTable.hpp"
void FieldInfo::print(outputStream* os, ConstantPool* cp) {
os->print_cr("index=%d name_index=%d name=%s signature_index=%d signature=%s offset=%d "
@@ -37,8 +40,10 @@ void FieldInfo::print(outputStream* os, ConstantPool* cp) {
field_flags().as_uint(),
initializer_index(),
generic_signature_index(),
- _field_flags.is_injected() ? lookup_symbol(generic_signature_index())->as_utf8() : cp->symbol_at(generic_signature_index())->as_utf8(),
- contended_group());
+ _field_flags.is_generic() ? (_field_flags.is_injected() ?
+ lookup_symbol(generic_signature_index())->as_utf8() : cp->symbol_at(generic_signature_index())->as_utf8()
+ ) : "",
+ is_contended() ? contended_group() : 0);
}
void FieldInfo::print_from_growable_array(outputStream* os, GrowableArray* array, ConstantPool* cp) {
@@ -62,13 +67,17 @@ Array* FieldInfoStream::create_FieldInfoStream(GrowableArray* fie
StreamSizer s;
StreamFieldSizer sizer(&s);
+ assert(fields->length() == java_fields + injected_fields, "must be");
+
sizer.consumer()->accept_uint(java_fields);
sizer.consumer()->accept_uint(injected_fields);
for (int i = 0; i < fields->length(); i++) {
FieldInfo* fi = fields->adr_at(i);
sizer.map_field_info(*fi);
}
- int storage_size = sizer.consumer()->position() + 1;
+ // Originally there was an extra byte with 0 terminating the reading;
+ // now we check limits instead.
+ int storage_size = sizer.consumer()->position();
Array* const fis = MetadataFactory::new_array(loader_data, storage_size, CHECK_NULL);
using StreamWriter = UNSIGNED5::Writer*, int, ArrayHelper*, int>>;
@@ -79,15 +88,14 @@ Array* FieldInfoStream::create_FieldInfoStream(GrowableArray* fie
writer.consumer()->accept_uint(java_fields);
writer.consumer()->accept_uint(injected_fields);
for (int i = 0; i < fields->length(); i++) {
- FieldInfo* fi = fields->adr_at(i);
- writer.map_field_info(*fi);
+ writer.map_field_info(fields->at(i));
}
#ifdef ASSERT
FieldInfoReader r(fis);
- int jfc = r.next_uint();
+ int jfc, ifc;
+ r.read_field_counts(&jfc, &ifc);
assert(jfc == java_fields, "Must be");
- int ifc = r.next_uint();
assert(ifc == injected_fields, "Must be");
for (int i = 0; i < jfc + ifc; i++) {
FieldInfo fi;
@@ -113,30 +121,221 @@ Array* FieldInfoStream::create_FieldInfoStream(GrowableArray* fie
return fis;
}
-GrowableArray* FieldInfoStream::create_FieldInfoArray(const Array* fis, int* java_fields_count, int* injected_fields_count) {
- int length = FieldInfoStream::num_total_fields(fis);
- GrowableArray* array = new GrowableArray(length);
+int FieldInfoStream::compare_name_and_sig(const Symbol* n1, const Symbol* s1, const Symbol* n2, const Symbol* s2) {
+ int cmp = n1->fast_compare(n2);
+ return cmp != 0 ? cmp : s1->fast_compare(s2);
+}
+
+
+// We use both name and signature during the comparison; while JLS require unique
+// names for fields, JVMS requires only unique name + signature combination.
+struct field_pos {
+ Symbol* _name;
+ Symbol* _signature;
+ int _index;
+ int _position;
+};
+
+class FieldInfoSupplier: public PackedTableBuilder::Supplier {
+ const field_pos* _positions;
+ size_t _elements;
+
+public:
+ FieldInfoSupplier(const field_pos* positions, size_t elements): _positions(positions), _elements(elements) {}
+
+ bool next(uint32_t* key, uint32_t* value) override {
+ if (_elements == 0) {
+ return false;
+ }
+ *key = _positions->_position;
+ *value = _positions->_index;
+ ++_positions;
+ --_elements;
+ return true;
+ }
+};
+
+Array* FieldInfoStream::create_search_table(ConstantPool* cp, const Array* fis, ClassLoaderData* loader_data, TRAPS) {
+ if (CDSConfig::is_dumping_dynamic_archive()) {
+ // We cannot use search table; in case of dynamic archives it should be sorted by "requested" addresses,
+ // but Symbol* addresses are coming from _constants, which has "buffered" addresses.
+ // For background, see new comments inside allocate_node_impl in symbolTable.cpp
+ return nullptr;
+ }
+
FieldInfoReader r(fis);
- *java_fields_count = r.next_uint();
- *injected_fields_count = r.next_uint();
+ int java_fields;
+ int injected_fields;
+ r.read_field_counts(&java_fields, &injected_fields);
+ assert(java_fields >= 0, "must be");
+ if (java_fields == 0 || fis->length() == 0 || static_cast(java_fields) < BinarySearchThreshold) {
+ return nullptr;
+ }
+
+ ResourceMark rm;
+ field_pos* positions = NEW_RESOURCE_ARRAY(field_pos, java_fields);
+ for (int i = 0; i < java_fields; ++i) {
+ assert(r.has_next(), "number of fields must match");
+
+ positions[i]._position = r.position();
+ FieldInfo fi;
+ r.read_field_info(fi);
+
+ positions[i]._name = fi.name(cp);
+ positions[i]._signature = fi.signature(cp);
+ positions[i]._index = i;
+ }
+ auto compare_pair = [](const void* v1, const void* v2) {
+ const field_pos* p1 = reinterpret_cast(v1);
+ const field_pos* p2 = reinterpret_cast(v2);
+ return compare_name_and_sig(p1->_name, p1->_signature, p2->_name, p2->_signature);
+ };
+ qsort(positions, java_fields, sizeof(field_pos), compare_pair);
+
+ PackedTableBuilder builder(fis->length() - 1, java_fields - 1);
+ Array* table = MetadataFactory::new_array(loader_data, java_fields * builder.element_bytes(), CHECK_NULL);
+ FieldInfoSupplier supplier(positions, java_fields);
+ builder.fill(table->data(), static_cast(table->length()), supplier);
+ return table;
+}
+
+GrowableArray* FieldInfoStream::create_FieldInfoArray(const Array* fis, int* java_fields_count, int* injected_fields_count) {
+ FieldInfoReader r(fis);
+ r.read_field_counts(java_fields_count, injected_fields_count);
+ int length = *java_fields_count + *injected_fields_count;
+
+ GrowableArray* array = new GrowableArray(length);
while (r.has_next()) {
FieldInfo fi;
r.read_field_info(fi);
array->append(fi);
}
assert(array->length() == length, "Must be");
- assert(array->length() == *java_fields_count + *injected_fields_count, "Must be");
return array;
}
void FieldInfoStream::print_from_fieldinfo_stream(Array* fis, outputStream* os, ConstantPool* cp) {
- int length = FieldInfoStream::num_total_fields(fis);
FieldInfoReader r(fis);
- int java_field_count = r.next_uint();
- int injected_fields_count = r.next_uint();
+ int java_fields_count;
+ int injected_fields_count;
+ r.read_field_counts(&java_fields_count, &injected_fields_count);
while (r.has_next()) {
FieldInfo fi;
r.read_field_info(fi);
fi.print(os, cp);
}
}
+
+class FieldInfoComparator: public PackedTableLookup::Comparator {
+ const FieldInfoReader* _reader;
+ ConstantPool* _cp;
+ const Symbol* _name;
+ const Symbol* _signature;
+
+public:
+ FieldInfoComparator(const FieldInfoReader* reader, ConstantPool* cp, const Symbol* name, const Symbol* signature):
+ _reader(reader), _cp(cp), _name(name), _signature(signature) {}
+
+ int compare_to(uint32_t position) override {
+ FieldInfoReader r2(*_reader);
+ r2.set_position_and_next_index(position, -1);
+ u2 name_index, sig_index;
+ r2.read_name_and_signature(&name_index, &sig_index);
+ Symbol* mid_name = _cp->symbol_at(name_index);
+ Symbol* mid_sig = _cp->symbol_at(sig_index);
+
+ return FieldInfoStream::compare_name_and_sig(_name, _signature, mid_name, mid_sig);
+ }
+
+#ifdef ASSERT
+ void reset(uint32_t position) override {
+ FieldInfoReader r2(*_reader);
+ r2.set_position_and_next_index(position, -1);
+ u2 name_index, signature_index;
+ r2.read_name_and_signature(&name_index, &signature_index);
+ _name = _cp->symbol_at(name_index);
+ _signature = _cp->symbol_at(signature_index);
+ }
+#endif // ASSERT
+};
+
+#ifdef ASSERT
+void FieldInfoStream::validate_search_table(ConstantPool* cp, const Array* fis, const Array* search_table) {
+ if (search_table == nullptr) {
+ return;
+ }
+ FieldInfoReader reader(fis);
+ int java_fields, injected_fields;
+ reader.read_field_counts(&java_fields, &injected_fields);
+ assert(java_fields > 0, "must be");
+
+ PackedTableLookup lookup(fis->length() - 1, java_fields - 1, search_table);
+ assert(lookup.element_bytes() * java_fields == static_cast(search_table->length()), "size does not match");
+
+ FieldInfoComparator comparator(&reader, cp, nullptr, nullptr);
+ // Check 1: assert that elements have the correct order based on the comparison function
+ lookup.validate_order(comparator);
+
+ // Check 2: Iterate through the original stream (not just search_table) and try if lookup works as expected
+ reader.set_position_and_next_index(0, 0);
+ reader.read_field_counts(&java_fields, &injected_fields);
+ while (reader.has_next()) {
+ int field_start = reader.position();
+ FieldInfo fi;
+ reader.read_field_info(fi);
+ if (fi.field_flags().is_injected()) {
+ // checking only java fields that precede injected ones
+ break;
+ }
+
+ FieldInfoReader r2(fis);
+ int index = r2.search_table_lookup(search_table, fi.name(cp), fi.signature(cp), cp, java_fields);
+ assert(index == static_cast(fi.index()), "wrong index: %d != %u", index, fi.index());
+ assert(index == r2.next_index(), "index should match");
+ assert(field_start == r2.position(), "must find the same position");
+ }
+}
+#endif // ASSERT
+
+void FieldInfoStream::print_search_table(outputStream* st, ConstantPool* cp, const Array* fis, const Array* search_table) {
+ if (search_table == nullptr) {
+ return;
+ }
+ FieldInfoReader reader(fis);
+ int java_fields, injected_fields;
+ reader.read_field_counts(&java_fields, &injected_fields);
+ assert(java_fields > 0, "must be");
+ PackedTableLookup lookup(fis->length() - 1, java_fields - 1, search_table);
+ auto printer = [&] (size_t offset, uint32_t position, uint32_t index) {
+ reader.set_position_and_next_index(position, -1);
+ u2 name_index, sig_index;
+ reader.read_name_and_signature(&name_index, &sig_index);
+ Symbol* name = cp->symbol_at(name_index);
+ Symbol* sig = cp->symbol_at(sig_index);
+ st->print(" [%zu] #%d,#%d = ", offset, name_index, sig_index);
+ name->print_symbol_on(st);
+ st->print(":");
+ sig->print_symbol_on(st);
+ st->print(" @ %p,%p", name, sig);
+ st->cr();
+ };
+
+ lookup.iterate(printer);
+}
+
+int FieldInfoReader::search_table_lookup(const Array* search_table, const Symbol* name, const Symbol* signature, ConstantPool* cp, int java_fields) {
+ assert(java_fields >= 0, "must be");
+ if (java_fields == 0) {
+ return -1;
+ }
+ FieldInfoComparator comp(this, cp, name, signature);
+ PackedTableLookup lookup(_r.limit() - 1, java_fields - 1, search_table);
+ uint32_t position;
+ static_assert(sizeof(uint32_t) == sizeof(_next_index), "field size assert");
+ if (lookup.search(comp, &position, reinterpret_cast(&_next_index))) {
+ _r.set_position(static_cast(position));
+ return _next_index;
+ } else {
+ return -1;
+ }
+}
diff --git a/src/hotspot/share/oops/fieldInfo.hpp b/src/hotspot/share/oops/fieldInfo.hpp
index 8740d539c8f..a98895da9cf 100644
--- a/src/hotspot/share/oops/fieldInfo.hpp
+++ b/src/hotspot/share/oops/fieldInfo.hpp
@@ -222,29 +222,28 @@ public:
void map_field_info(const FieldInfo& fi);
};
-
// Gadget for decoding and reading the stream of field records.
class FieldInfoReader {
- friend class FieldInfoStream;
- friend class ClassFileParser;
- friend class FieldStreamBase;
- friend class FieldInfo;
-
UNSIGNED5::Reader _r;
int _next_index;
- public:
+public:
FieldInfoReader(const Array* fi);
- private:
- uint32_t next_uint() { return _r.next_uint(); }
+private:
+ inline uint32_t next_uint() { return _r.next_uint(); }
void skip(int n) { int s = _r.try_skip(n); assert(s == n,""); }
public:
- int has_next() { return _r.has_next(); }
- int position() { return _r.position(); }
- int next_index() { return _next_index; }
+ void read_field_counts(int* java_fields, int* injected_fields);
+ int has_next() const { return _r.position() < _r.limit(); }
+ int position() const { return _r.position(); }
+ int next_index() const { return _next_index; }
+ void read_name_and_signature(u2* name_index, u2* signature_index);
void read_field_info(FieldInfo& fi);
+
+ int search_table_lookup(const Array* search_table, const Symbol* name, const Symbol* signature, ConstantPool* cp, int java_fields);
+
// skip a whole field record, both required and optional bits
FieldInfoReader& skip_field_info();
@@ -271,6 +270,11 @@ class FieldInfoStream : AllStatic {
friend class JavaFieldStream;
friend class FieldStreamBase;
friend class ClassFileParser;
+ friend class FieldInfoReader;
+ friend class FieldInfoComparator;
+
+ private:
+ static int compare_name_and_sig(const Symbol* n1, const Symbol* s1, const Symbol* n2, const Symbol* s2);
public:
static int num_java_fields(const Array* fis);
@@ -278,9 +282,14 @@ class FieldInfoStream : AllStatic {
static int num_total_fields(const Array* fis);
static Array* create_FieldInfoStream(GrowableArray* fields, int java_fields, int injected_fields,
- ClassLoaderData* loader_data, TRAPS);
+ ClassLoaderData* loader_data, TRAPS);
+ static Array* create_search_table(ConstantPool* cp, const Array* fis, ClassLoaderData* loader_data, TRAPS);
static GrowableArray* create_FieldInfoArray(const Array* fis, int* java_fields_count, int* injected_fields_count);
static void print_from_fieldinfo_stream(Array* fis, outputStream* os, ConstantPool* cp);
+
+ DEBUG_ONLY(static void validate_search_table(ConstantPool* cp, const Array* fis, const Array* search_table);)
+
+ static void print_search_table(outputStream* st, ConstantPool* cp, const Array* fis, const Array* search_table);
};
class FieldStatus {
diff --git a/src/hotspot/share/oops/fieldInfo.inline.hpp b/src/hotspot/share/oops/fieldInfo.inline.hpp
index d3d4d765081..842393729b2 100644
--- a/src/hotspot/share/oops/fieldInfo.inline.hpp
+++ b/src/hotspot/share/oops/fieldInfo.inline.hpp
@@ -56,16 +56,27 @@ inline Symbol* FieldInfo::lookup_symbol(int symbol_index) const {
inline int FieldInfoStream::num_injected_java_fields(const Array* fis) {
FieldInfoReader fir(fis);
- fir.skip(1);
- return fir.next_uint();
+ int java_fields_count;
+ int injected_fields_count;
+ fir.read_field_counts(&java_fields_count, &injected_fields_count);
+ return injected_fields_count;
}
inline int FieldInfoStream::num_total_fields(const Array* fis) {
FieldInfoReader fir(fis);
- return fir.next_uint() + fir.next_uint();
+ int java_fields_count;
+ int injected_fields_count;
+ fir.read_field_counts(&java_fields_count, &injected_fields_count);
+ return java_fields_count + injected_fields_count;
}
-inline int FieldInfoStream::num_java_fields(const Array* fis) { return FieldInfoReader(fis).next_uint(); }
+inline int FieldInfoStream::num_java_fields(const Array* fis) {
+ FieldInfoReader fir(fis);
+ int java_fields_count;
+ int injected_fields_count;
+ fir.read_field_counts(&java_fields_count, &injected_fields_count);
+ return java_fields_count;
+}
template
inline void Mapper::map_field_info(const FieldInfo& fi) {
@@ -94,13 +105,22 @@ inline void Mapper::map_field_info(const FieldInfo& fi) {
inline FieldInfoReader::FieldInfoReader(const Array* fi)
- : _r(fi->data(), 0),
+ : _r(fi->data(), fi->length()),
_next_index(0) { }
+inline void FieldInfoReader::read_field_counts(int* java_fields, int* injected_fields) {
+ *java_fields = next_uint();
+ *injected_fields = next_uint();
+}
+
+inline void FieldInfoReader::read_name_and_signature(u2* name_index, u2* signature_index) {
+ *name_index = checked_cast(next_uint());
+ *signature_index = checked_cast(next_uint());
+}
+
inline void FieldInfoReader::read_field_info(FieldInfo& fi) {
fi._index = _next_index++;
- fi._name_index = checked_cast(next_uint());
- fi._signature_index = checked_cast(next_uint());
+ read_name_and_signature(&fi._name_index, &fi._signature_index);
fi._offset = next_uint();
fi._access_flags = AccessFlags(checked_cast(next_uint()));
fi._field_flags = FieldInfo::FieldFlags(next_uint());
diff --git a/src/hotspot/share/oops/fieldStreams.hpp b/src/hotspot/share/oops/fieldStreams.hpp
index a1c5d77eeb6..0ae828d73d9 100644
--- a/src/hotspot/share/oops/fieldStreams.hpp
+++ b/src/hotspot/share/oops/fieldStreams.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,17 +56,23 @@ class FieldStreamBase : public StackObj {
inline FieldStreamBase(const Array* fieldinfo_stream, ConstantPool* constants, int start, int limit);
- inline FieldStreamBase(Array* fieldinfo_stream, ConstantPool* constants);
+ inline FieldStreamBase(const Array* fieldinfo_stream, ConstantPool* constants);
- private:
+ private:
void initialize() {
- int java_fields_count = _reader.next_uint();
- int injected_fields_count = _reader.next_uint();
- assert( _limit <= java_fields_count + injected_fields_count, "Safety check");
+ int java_fields_count;
+ int injected_fields_count;
+ _reader.read_field_counts(&java_fields_count, &injected_fields_count);
+ if (_limit < _index) {
+ _limit = java_fields_count + injected_fields_count;
+ } else {
+ assert( _limit <= java_fields_count + injected_fields_count, "Safety check");
+ }
if (_limit != 0) {
_reader.read_field_info(_fi_buf);
}
}
+
public:
inline FieldStreamBase(InstanceKlass* klass);
@@ -138,8 +144,11 @@ class FieldStreamBase : public StackObj {
// Iterate over only the Java fields
class JavaFieldStream : public FieldStreamBase {
+ Array* _search_table;
+
public:
- JavaFieldStream(const InstanceKlass* k): FieldStreamBase(k->fieldinfo_stream(), k->constants(), 0, k->java_fields_count()) {}
+ JavaFieldStream(const InstanceKlass* k): FieldStreamBase(k->fieldinfo_stream(), k->constants(), 0, k->java_fields_count()),
+ _search_table(k->fieldinfo_search_table()) {}
u2 name_index() const {
assert(!field()->field_flags().is_injected(), "regular only");
@@ -149,7 +158,6 @@ class JavaFieldStream : public FieldStreamBase {
u2 signature_index() const {
assert(!field()->field_flags().is_injected(), "regular only");
return field()->signature_index();
- return -1;
}
u2 generic_signature_index() const {
@@ -164,6 +172,10 @@ class JavaFieldStream : public FieldStreamBase {
assert(!field()->field_flags().is_injected(), "regular only");
return field()->initializer_index();
}
+
+ // Performs either a linear search or binary search through the stream
+ // looking for a matching name/signature combo
+ bool lookup(const Symbol* name, const Symbol* signature);
};
@@ -176,7 +188,6 @@ class InternalFieldStream : public FieldStreamBase {
class AllFieldStream : public FieldStreamBase {
public:
- AllFieldStream(Array* fieldinfo, ConstantPool* constants): FieldStreamBase(fieldinfo, constants) {}
AllFieldStream(const InstanceKlass* k): FieldStreamBase(k->fieldinfo_stream(), k->constants()) {}
};
diff --git a/src/hotspot/share/oops/fieldStreams.inline.hpp b/src/hotspot/share/oops/fieldStreams.inline.hpp
index 776bcd1671c..51faf88c678 100644
--- a/src/hotspot/share/oops/fieldStreams.inline.hpp
+++ b/src/hotspot/share/oops/fieldStreams.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,22 +33,18 @@
FieldStreamBase::FieldStreamBase(const Array* fieldinfo_stream, ConstantPool* constants, int start, int limit) :
_fieldinfo_stream(fieldinfo_stream),
_reader(FieldInfoReader(_fieldinfo_stream)),
- _constants(constantPoolHandle(Thread::current(), constants)), _index(start) {
- _index = start;
- if (limit < start) {
- _limit = FieldInfoStream::num_total_fields(_fieldinfo_stream);
- } else {
- _limit = limit;
- }
+ _constants(constantPoolHandle(Thread::current(), constants)),
+ _index(start),
+ _limit(limit) {
initialize();
}
-FieldStreamBase::FieldStreamBase(Array* fieldinfo_stream, ConstantPool* constants) :
+FieldStreamBase::FieldStreamBase(const Array* fieldinfo_stream, ConstantPool* constants) :
_fieldinfo_stream(fieldinfo_stream),
_reader(FieldInfoReader(_fieldinfo_stream)),
_constants(constantPoolHandle(Thread::current(), constants)),
_index(0),
- _limit(FieldInfoStream::num_total_fields(_fieldinfo_stream)) {
+ _limit(-1) {
initialize();
}
@@ -57,9 +53,28 @@ FieldStreamBase::FieldStreamBase(InstanceKlass* klass) :
_reader(FieldInfoReader(_fieldinfo_stream)),
_constants(constantPoolHandle(Thread::current(), klass->constants())),
_index(0),
- _limit(FieldInfoStream::num_total_fields(_fieldinfo_stream)) {
+ _limit(-1) {
assert(klass == field_holder(), "");
initialize();
}
+inline bool JavaFieldStream::lookup(const Symbol* name, const Symbol* signature) {
+ if (_search_table != nullptr) {
+ int index = _reader.search_table_lookup(_search_table, name, signature, _constants(), _limit);
+ if (index >= 0) {
+ assert(index < _limit, "must be");
+ _index = index;
+ _reader.read_field_info(_fi_buf);
+ return true;
+ }
+ } else {
+ for (; !done(); next()) {
+ if (this->name() == name && this->signature() == signature) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
#endif // SHARE_OOPS_FIELDSTREAMS_INLINE_HPP
diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp
index b3d896b6863..1bb6b3bdf98 100644
--- a/src/hotspot/share/oops/instanceKlass.cpp
+++ b/src/hotspot/share/oops/instanceKlass.cpp
@@ -686,6 +686,11 @@ void InstanceKlass::deallocate_contents(ClassLoaderData* loader_data) {
}
set_fieldinfo_stream(nullptr);
+ if (fieldinfo_search_table() != nullptr && !fieldinfo_search_table()->is_shared()) {
+ MetadataFactory::free_array(loader_data, fieldinfo_search_table());
+ }
+ set_fieldinfo_search_table(nullptr);
+
if (fields_status() != nullptr && !fields_status()->is_shared()) {
MetadataFactory::free_array(loader_data, fields_status());
}
@@ -1786,13 +1791,12 @@ FieldInfo InstanceKlass::field(int index) const {
}
bool InstanceKlass::find_local_field(Symbol* name, Symbol* sig, fieldDescriptor* fd) const {
- for (JavaFieldStream fs(this); !fs.done(); fs.next()) {
- Symbol* f_name = fs.name();
- Symbol* f_sig = fs.signature();
- if (f_name == name && f_sig == sig) {
- fd->reinitialize(const_cast(this), fs.to_FieldInfo());
- return true;
- }
+ JavaFieldStream fs(this);
+ if (fs.lookup(name, sig)) {
+ assert(fs.name() == name, "name must match");
+ assert(fs.signature() == sig, "signature must match");
+ fd->reinitialize(const_cast(this), fs.to_FieldInfo());
+ return true;
}
return false;
}
@@ -2610,6 +2614,7 @@ void InstanceKlass::metaspace_pointers_do(MetaspaceClosure* it) {
}
it->push(&_fieldinfo_stream);
+ it->push(&_fieldinfo_search_table);
// _fields_status might be written into by Rewriter::scan_method() -> fd.set_has_initialized_final_update()
it->push(&_fields_status, MetaspaceClosure::_writable);
@@ -2710,6 +2715,8 @@ void InstanceKlass::remove_unshareable_info() {
DEBUG_ONLY(_shared_class_load_count = 0);
remove_unshareable_flags();
+
+ DEBUG_ONLY(FieldInfoStream::validate_search_table(_constants, _fieldinfo_stream, _fieldinfo_search_table));
}
void InstanceKlass::remove_unshareable_flags() {
@@ -2816,6 +2823,8 @@ void InstanceKlass::restore_unshareable_info(ClassLoaderData* loader_data, Handl
if (DiagnoseSyncOnValueBasedClasses && has_value_based_class_annotation() && !is_value_based()) {
set_is_value_based();
}
+
+ DEBUG_ONLY(FieldInfoStream::validate_search_table(_constants, _fieldinfo_stream, _fieldinfo_search_table));
}
// Check if a class or any of its supertypes has a version older than 50.
@@ -3492,7 +3501,7 @@ void InstanceKlass::add_osr_nmethod(nmethod* n) {
for (int l = CompLevel_limited_profile; l < n->comp_level(); l++) {
nmethod *inv = lookup_osr_nmethod(n->method(), n->osr_entry_bci(), l, true);
if (inv != nullptr && inv->is_in_use()) {
- inv->make_not_entrant("OSR invalidation of lower levels");
+ inv->make_not_entrant(nmethod::ChangeReason::OSR_invalidation_of_lower_level);
}
}
}
@@ -3760,6 +3769,11 @@ void InstanceKlass::print_on(outputStream* st) const {
map++;
}
st->cr();
+
+ if (fieldinfo_search_table() != nullptr) {
+ st->print_cr(BULLET"---- field info search table:");
+ FieldInfoStream::print_search_table(st, _constants, _fieldinfo_stream, _fieldinfo_search_table);
+ }
}
void InstanceKlass::print_value_on(outputStream* st) const {
diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp
index 078c5b81841..55ab0996a4a 100644
--- a/src/hotspot/share/oops/instanceKlass.hpp
+++ b/src/hotspot/share/oops/instanceKlass.hpp
@@ -276,6 +276,7 @@ class InstanceKlass: public Klass {
// Fields information is stored in an UNSIGNED5 encoded stream (see fieldInfo.hpp)
Array* _fieldinfo_stream;
+ Array* _fieldinfo_search_table;
Array* _fields_status;
// embedded Java vtable follows here
@@ -398,6 +399,9 @@ class InstanceKlass: public Klass {
Array* fieldinfo_stream() const { return _fieldinfo_stream; }
void set_fieldinfo_stream(Array* fis) { _fieldinfo_stream = fis; }
+ Array* fieldinfo_search_table() const { return _fieldinfo_search_table; }
+ void set_fieldinfo_search_table(Array* table) { _fieldinfo_search_table = table; }
+
Array* fields_status() const {return _fields_status; }
void set_fields_status(Array* array) { _fields_status = array; }
diff --git a/src/hotspot/share/oops/metadata.hpp b/src/hotspot/share/oops/metadata.hpp
index ff18cef60a7..5a890a27fa1 100644
--- a/src/hotspot/share/oops/metadata.hpp
+++ b/src/hotspot/share/oops/metadata.hpp
@@ -44,6 +44,7 @@ class Metadata : public MetaspaceObj {
virtual bool is_method() const { return false; }
virtual bool is_methodData() const { return false; }
virtual bool is_constantPool() const { return false; }
+ virtual bool is_methodCounters() const { return false; }
virtual int size() const = 0;
virtual MetaspaceObj::Type type() const = 0;
virtual const char* internal_name() const = 0;
diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp
index d13458dfa93..1a36fce23aa 100644
--- a/src/hotspot/share/oops/method.cpp
+++ b/src/hotspot/share/oops/method.cpp
@@ -1028,7 +1028,7 @@ void Method::set_native_function(address function, bool post_event_flag) {
// If so, we have to make it not_entrant.
nmethod* nm = code(); // Put it into local variable to guard against concurrent updates
if (nm != nullptr) {
- nm->make_not_entrant("set native function");
+ nm->make_not_entrant(nmethod::ChangeReason::set_native_function);
}
}
diff --git a/src/hotspot/share/oops/objArrayKlass.cpp b/src/hotspot/share/oops/objArrayKlass.cpp
index 7af6b963052..e1fcc25f150 100644
--- a/src/hotspot/share/oops/objArrayKlass.cpp
+++ b/src/hotspot/share/oops/objArrayKlass.cpp
@@ -44,7 +44,7 @@
#include "runtime/mutexLocker.hpp"
#include "utilities/macros.hpp"
-ObjArrayKlass* ObjArrayKlass::allocate(ClassLoaderData* loader_data, int n, Klass* k, Symbol* name, TRAPS) {
+ObjArrayKlass* ObjArrayKlass::allocate_klass(ClassLoaderData* loader_data, int n, Klass* k, Symbol* name, TRAPS) {
assert(ObjArrayKlass::header_size() <= InstanceKlass::header_size(),
"array klasses must be same size as InstanceKlass");
@@ -100,7 +100,7 @@ ObjArrayKlass* ObjArrayKlass::allocate_objArray_klass(ClassLoaderData* loader_da
}
// Initialize instance variables
- ObjArrayKlass* oak = ObjArrayKlass::allocate(loader_data, n, element_klass, name, CHECK_NULL);
+ ObjArrayKlass* oak = ObjArrayKlass::allocate_klass(loader_data, n, element_klass, name, CHECK_NULL);
ModuleEntry* module = oak->module();
assert(module != nullptr, "No module entry for array");
@@ -149,7 +149,7 @@ size_t ObjArrayKlass::oop_size(oop obj) const {
return objArrayOop(obj)->object_size();
}
-objArrayOop ObjArrayKlass::allocate(int length, TRAPS) {
+objArrayOop ObjArrayKlass::allocate_instance(int length, TRAPS) {
check_array_allocation_length(length, arrayOopDesc::max_array_length(T_OBJECT), CHECK_NULL);
size_t size = objArrayOopDesc::object_size(length);
return (objArrayOop)Universe::heap()->array_allocate(this, size, length,
@@ -160,7 +160,7 @@ oop ObjArrayKlass::multi_allocate(int rank, jint* sizes, TRAPS) {
int length = *sizes;
ArrayKlass* ld_klass = lower_dimension();
// If length < 0 allocate will throw an exception.
- objArrayOop array = allocate(length, CHECK_NULL);
+ objArrayOop array = allocate_instance(length, CHECK_NULL);
objArrayHandle h_array (THREAD, array);
if (rank > 1) {
if (length != 0) {
diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp
index c44c8d28f62..11fe4f2a521 100644
--- a/src/hotspot/share/oops/objArrayKlass.hpp
+++ b/src/hotspot/share/oops/objArrayKlass.hpp
@@ -33,8 +33,10 @@ class ClassLoaderData;
// ObjArrayKlass is the klass for objArrays
class ObjArrayKlass : public ArrayKlass {
- friend class VMStructs;
+ friend class Deoptimization;
friend class JVMCIVMStructs;
+ friend class oopFactory;
+ friend class VMStructs;
public:
static const KlassKind Kind = ObjArrayKlassKind;
@@ -47,7 +49,9 @@ class ObjArrayKlass : public ArrayKlass {
// Constructor
ObjArrayKlass(int n, Klass* element_klass, Symbol* name);
- static ObjArrayKlass* allocate(ClassLoaderData* loader_data, int n, Klass* k, Symbol* name, TRAPS);
+ static ObjArrayKlass* allocate_klass(ClassLoaderData* loader_data, int n, Klass* k, Symbol* name, TRAPS);
+
+ objArrayOop allocate_instance(int length, TRAPS);
public:
// For dummy objects
ObjArrayKlass() {}
@@ -78,7 +82,6 @@ class ObjArrayKlass : public ArrayKlass {
static ObjArrayKlass* allocate_objArray_klass(ClassLoaderData* loader_data,
int n, Klass* element_klass, TRAPS);
- objArrayOop allocate(int length, TRAPS);
oop multi_allocate(int rank, jint* sizes, TRAPS);
// Copying
diff --git a/src/hotspot/share/oops/trainingData.cpp b/src/hotspot/share/oops/trainingData.cpp
index 543462b1182..2ae1ac18699 100644
--- a/src/hotspot/share/oops/trainingData.cpp
+++ b/src/hotspot/share/oops/trainingData.cpp
@@ -432,24 +432,11 @@ void KlassTrainingData::print_on(outputStream* st, bool name_only) const {
}
KlassTrainingData::KlassTrainingData(InstanceKlass* klass) : TrainingData(klass) {
- if (holder() == klass) {
- return; // no change to make
- }
-
- jobject hmj = _holder_mirror;
- if (hmj != nullptr) { // clear out previous handle, if any
- _holder_mirror = nullptr;
- assert(JNIHandles::is_global_handle(hmj), "");
- JNIHandles::destroy_global(hmj);
- }
-
- if (klass != nullptr) {
- Handle hm(JavaThread::current(), klass->java_mirror());
- hmj = JNIHandles::make_global(hm);
- Atomic::release_store(&_holder_mirror, hmj);
- }
-
- Atomic::release_store(&_holder, const_cast(klass));
+ assert(klass != nullptr, "");
+ Handle hm(JavaThread::current(), klass->java_mirror());
+ jobject hmj = JNIHandles::make_global(hm);
+ _holder_mirror = hmj;
+ _holder = klass;
assert(holder() == klass, "");
}
diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp
index 39385bb0184..bdf37c7db49 100644
--- a/src/hotspot/share/oops/typeArrayKlass.cpp
+++ b/src/hotspot/share/oops/typeArrayKlass.cpp
@@ -50,7 +50,7 @@ TypeArrayKlass* TypeArrayKlass::create_klass(BasicType type,
ClassLoaderData* null_loader_data = ClassLoaderData::the_null_class_loader_data();
- TypeArrayKlass* ak = TypeArrayKlass::allocate(null_loader_data, type, sym, CHECK_NULL);
+ TypeArrayKlass* ak = TypeArrayKlass::allocate_klass(null_loader_data, type, sym, CHECK_NULL);
// Call complete_create_array_klass after all instance variables have been initialized.
complete_create_array_klass(ak, ak->super(), ModuleEntryTable::javabase_moduleEntry(), CHECK_NULL);
@@ -65,7 +65,7 @@ TypeArrayKlass* TypeArrayKlass::create_klass(BasicType type,
return ak;
}
-TypeArrayKlass* TypeArrayKlass::allocate(ClassLoaderData* loader_data, BasicType type, Symbol* name, TRAPS) {
+TypeArrayKlass* TypeArrayKlass::allocate_klass(ClassLoaderData* loader_data, BasicType type, Symbol* name, TRAPS) {
assert(TypeArrayKlass::header_size() <= InstanceKlass::header_size(),
"array klasses must be same size as InstanceKlass");
@@ -101,7 +101,7 @@ oop TypeArrayKlass::multi_allocate(int rank, jint* last_size, TRAPS) {
// For typeArrays this is only called for the last dimension
assert(rank == 1, "just checking");
int length = *last_size;
- return allocate(length, THREAD);
+ return allocate_instance(length, THREAD);
}
diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp
index fa4e301e3e4..22600702fe2 100644
--- a/src/hotspot/share/oops/typeArrayKlass.hpp
+++ b/src/hotspot/share/oops/typeArrayKlass.hpp
@@ -33,6 +33,8 @@ class ClassLoaderData;
// It contains the type and size of the elements
class TypeArrayKlass : public ArrayKlass {
+ friend class Deoptimization;
+ friend class oopFactory;
friend class VMStructs;
public:
@@ -43,7 +45,10 @@ class TypeArrayKlass : public ArrayKlass {
// Constructor
TypeArrayKlass(BasicType type, Symbol* name);
- static TypeArrayKlass* allocate(ClassLoaderData* loader_data, BasicType type, Symbol* name, TRAPS);
+ static TypeArrayKlass* allocate_klass(ClassLoaderData* loader_data, BasicType type, Symbol* name, TRAPS);
+
+ typeArrayOop allocate_common(int length, bool do_zero, TRAPS);
+ typeArrayOop allocate_instance(int length, TRAPS) { return allocate_common(length, true, THREAD); }
public:
TypeArrayKlass() {} // For dummy objects.
@@ -66,8 +71,6 @@ class TypeArrayKlass : public ArrayKlass {
size_t oop_size(oop obj) const;
// Allocation
- typeArrayOop allocate_common(int length, bool do_zero, TRAPS);
- typeArrayOop allocate(int length, TRAPS) { return allocate_common(length, true, THREAD); }
oop multi_allocate(int rank, jint* sizes, TRAPS);
oop protection_domain() const { return nullptr; }
diff --git a/src/hotspot/share/opto/block.hpp b/src/hotspot/share/opto/block.hpp
index c5a3a589407..e6a98c28e77 100644
--- a/src/hotspot/share/opto/block.hpp
+++ b/src/hotspot/share/opto/block.hpp
@@ -464,6 +464,14 @@ class PhaseCFG : public Phase {
Node* catch_cleanup_find_cloned_def(Block* use_blk, Node* def, Block* def_blk, int n_clone_idx);
void catch_cleanup_inter_block(Node *use, Block *use_blk, Node *def, Block *def_blk, int n_clone_idx);
+ // Ensure that n happens at b or above, i.e. at a block that dominates b.
+ // We expect n to be an orphan node without further inputs.
+ void ensure_node_is_at_block_or_above(Node* n, Block* b);
+
+ // Move node n from its current placement into the end of block b.
+ // Move also outgoing Mach projections.
+ void move_node_and_its_projections_to_block(Node* n, Block* b);
+
// Detect implicit-null-check opportunities. Basically, find null checks
// with suitable memory ops nearby. Use the memory op to do the null check.
// I can generate a memory op if there is not one nearby.
diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp
index fd55f2fd666..227817612ce 100644
--- a/src/hotspot/share/opto/c2_globals.hpp
+++ b/src/hotspot/share/opto/c2_globals.hpp
@@ -678,10 +678,12 @@
"Print progress during Iterative Global Value Numbering") \
\
develop(uint, VerifyIterativeGVN, 0, \
- "Verify Iterative Global Value Numbering" \
- "=XY, with Y: verify Def-Use modifications during IGVN" \
- " X: verify that type(n) == n->Value() after IGVN" \
- "X and Y in 0=off; 1=on") \
+ "Verify Iterative Global Value Numbering =DCBA, with:" \
+ " D: verify Node::Identity did not miss opportunities" \
+ " C: verify Node::Ideal did not miss opportunities" \
+ " B: verify that type(n) == n->Value() after IGVN" \
+ " A: verify Def-Use modifications during IGVN" \
+ "Each can be 0=off or 1=on") \
constraint(VerifyIterativeGVNConstraintFunc, AtParse) \
\
develop(bool, TraceCISCSpill, false, \
diff --git a/src/hotspot/share/opto/convertnode.cpp b/src/hotspot/share/opto/convertnode.cpp
index 5df79a36edb..c468c66fdfd 100644
--- a/src/hotspot/share/opto/convertnode.cpp
+++ b/src/hotspot/share/opto/convertnode.cpp
@@ -79,6 +79,11 @@ Node* Conv2BNode::Ideal(PhaseGVN* phase, bool can_reshape) {
assert(false, "Unrecognized comparison for Conv2B: %s", NodeClassNames[in(1)->Opcode()]);
}
+ // Skip the transformation if input is unexpected.
+ if (cmp == nullptr) {
+ return nullptr;
+ }
+
// Replace Conv2B with the cmove
Node* bol = phase->transform(new BoolNode(cmp, BoolTest::eq));
return new CMoveINode(bol, phase->intcon(1), phase->intcon(0), TypeInt::BOOL);
diff --git a/src/hotspot/share/opto/idealGraphPrinter.cpp b/src/hotspot/share/opto/idealGraphPrinter.cpp
index 3721394c112..1ecb46eaf5a 100644
--- a/src/hotspot/share/opto/idealGraphPrinter.cpp
+++ b/src/hotspot/share/opto/idealGraphPrinter.cpp
@@ -155,7 +155,6 @@ void IdealGraphPrinter::init(const char* file_name, bool use_multiple_files, boo
// in the mach where kill projections have no users but should
// appear in the dump.
_traverse_outs = true;
- _should_send_method = true;
_output = nullptr;
buffer[0] = 0;
_depth = 0;
@@ -300,13 +299,11 @@ void IdealGraphPrinter::print_inline_tree(InlineTree *tree) {
void IdealGraphPrinter::print_inlining() {
// Print inline tree
- if (_should_send_method) {
- InlineTree *inlineTree = C->ilt();
- if (inlineTree != nullptr) {
- print_inline_tree(inlineTree);
- } else {
- // print this method only
- }
+ InlineTree *inlineTree = C->ilt();
+ if (inlineTree != nullptr) {
+ print_inline_tree(inlineTree);
+ } else {
+ // print this method only
}
}
@@ -382,7 +379,6 @@ void IdealGraphPrinter::begin_method() {
tail(PROPERTIES_ELEMENT);
- _should_send_method = true;
this->_current_method = method;
_xml->flush();
@@ -975,7 +971,7 @@ void IdealGraphPrinter::print_graph(const char* name, const frame* fr) {
// Print current ideal graph
void IdealGraphPrinter::print(const char* name, Node* node, GrowableArray& visible_nodes, const frame* fr) {
- if (!_current_method || !_should_send_method || node == nullptr) return;
+ if (!_current_method || node == nullptr) return;
if (name == nullptr) {
stringStream graph_name;
diff --git a/src/hotspot/share/opto/idealGraphPrinter.hpp b/src/hotspot/share/opto/idealGraphPrinter.hpp
index 69ba2841506..7e68ce6c00f 100644
--- a/src/hotspot/share/opto/idealGraphPrinter.hpp
+++ b/src/hotspot/share/opto/idealGraphPrinter.hpp
@@ -110,7 +110,6 @@ class IdealGraphPrinter : public CHeapObj {
ciMethod *_current_method;
int _depth;
char buffer[2048];
- bool _should_send_method;
PhaseChaitin* _chaitin;
bool _traverse_outs;
Compile *C;
diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp
index 4df3bbab731..0fe246a7679 100644
--- a/src/hotspot/share/opto/lcm.cpp
+++ b/src/hotspot/share/opto/lcm.cpp
@@ -76,6 +76,36 @@ static bool needs_explicit_null_check_for_read(Node *val) {
return true;
}
+void PhaseCFG::move_node_and_its_projections_to_block(Node* n, Block* b) {
+ assert(!is_CFG(n), "cannot move CFG node");
+ Block* old = get_block_for_node(n);
+ old->find_remove(n);
+ b->add_inst(n);
+ map_node_to_block(n, b);
+ // Check for Mach projections that also need to be moved.
+ for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+ Node* out = n->fast_out(i);
+ if (!out->is_MachProj()) {
+ continue;
+ }
+ assert(!n->is_MachProj(), "nested projections are not allowed");
+ move_node_and_its_projections_to_block(out, b);
+ }
+}
+
+void PhaseCFG::ensure_node_is_at_block_or_above(Node* n, Block* b) {
+ assert(!is_CFG(n), "cannot move CFG node");
+ Block* current = get_block_for_node(n);
+ if (current->dominates(b)) {
+ return; // n is already placed above b, do nothing.
+ }
+ // We only expect nodes without further inputs, like MachTemp or load Base.
+ assert(n->req() == 0 || (n->req() == 1 && n->in(0) == (Node*)C->root()),
+ "need for recursive hoisting not expected");
+ assert(b->dominates(current), "precondition: can only move n to b if b dominates n");
+ move_node_and_its_projections_to_block(n, b);
+}
+
//------------------------------implicit_null_check----------------------------
// Detect implicit-null-check opportunities. Basically, find null checks
// with suitable memory ops nearby. Use the memory op to do the null check.
@@ -160,12 +190,14 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo
Node *m = val->out(i);
if( !m->is_Mach() ) continue;
MachNode *mach = m->as_Mach();
- if (mach->barrier_data() != 0) {
+ if (mach->barrier_data() != 0 &&
+ !mach->is_late_expanded_null_check_candidate()) {
// Using memory accesses with barriers to perform implicit null checks is
- // not supported. These operations might expand into multiple assembly
- // instructions during code emission, including new memory accesses (e.g.
- // in G1's pre-barrier), which would invalidate the implicit null
- // exception table.
+ // only supported if these are explicit marked as emitting a candidate
+ // memory access instruction at their initial address. If not marked as
+ // such, barrier-tagged operations might expand into one or several memory
+ // access instructions located at arbitrary offsets from the initial
+ // address, which would invalidate the implicit null exception table.
continue;
}
was_store = false;
@@ -321,6 +353,14 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo
// Ignore DecodeN val which could be hoisted to where needed.
if( is_decoden ) continue;
}
+ if (mach->in(j)->is_MachTemp()) {
+ assert(mach->in(j)->outcnt() == 1, "MachTemp nodes should not be shared");
+ // Ignore MachTemp inputs, they can be safely hoisted with the candidate.
+ // MachTemp nodes have no inputs themselves and are only used to reserve
+ // a scratch register for the implementation of the node (e.g. in
+ // late-expanded GC barriers).
+ continue;
+ }
// Block of memory-op input
Block *inb = get_block_for_node(mach->in(j));
Block *b = block; // Start from nul check
@@ -388,38 +428,24 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo
// Hoist it up to the end of the test block together with its inputs if they exist.
for (uint i = 2; i < val->req(); i++) {
// DecodeN has 2 regular inputs + optional MachTemp or load Base inputs.
- Node *temp = val->in(i);
- Block *tempb = get_block_for_node(temp);
- if (!tempb->dominates(block)) {
- assert(block->dominates(tempb), "sanity check: temp node placement");
- // We only expect nodes without further inputs, like MachTemp or load Base.
- assert(temp->req() == 0 || (temp->req() == 1 && temp->in(0) == (Node*)C->root()),
- "need for recursive hoisting not expected");
- tempb->find_remove(temp);
- block->add_inst(temp);
- map_node_to_block(temp, block);
- }
- }
- valb->find_remove(val);
- block->add_inst(val);
- map_node_to_block(val, block);
- // DecodeN on x86 may kill flags. Check for flag-killing projections
- // that also need to be hoisted.
- for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
- Node* n = val->fast_out(j);
- if( n->is_MachProj() ) {
- get_block_for_node(n)->find_remove(n);
- block->add_inst(n);
- map_node_to_block(n, block);
- }
+ // Inputs of val may already be early enough, but if not move them together with val.
+ ensure_node_is_at_block_or_above(val->in(i), block);
}
+ move_node_and_its_projections_to_block(val, block);
}
}
+
+ // Move any MachTemp inputs to the end of the test block.
+ for (uint i = 0; i < best->req(); i++) {
+ Node* n = best->in(i);
+ if (n == nullptr || !n->is_MachTemp()) {
+ continue;
+ }
+ ensure_node_is_at_block_or_above(n, block);
+ }
+
// Hoist the memory candidate up to the end of the test block.
- Block *old_block = get_block_for_node(best);
- old_block->find_remove(best);
- block->add_inst(best);
- map_node_to_block(best, block);
+ move_node_and_its_projections_to_block(best, block);
// Move the control dependence if it is pinned to not-null block.
// Don't change it in other cases: null or dominating control.
@@ -429,17 +455,6 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo
best->set_req(0, proj->in(0)->in(0));
}
- // Check for flag-killing projections that also need to be hoisted
- // Should be DU safe because no edge updates.
- for (DUIterator_Fast jmax, j = best->fast_outs(jmax); j < jmax; j++) {
- Node* n = best->fast_out(j);
- if( n->is_MachProj() ) {
- get_block_for_node(n)->find_remove(n);
- block->add_inst(n);
- map_node_to_block(n, block);
- }
- }
-
// proj==Op_True --> ne test; proj==Op_False --> eq test.
// One of two graph shapes got matched:
// (IfTrue (If (Bool NE (CmpP ptr null))))
diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp
index 29f737bce08..e9878cc0c77 100644
--- a/src/hotspot/share/opto/library_call.cpp
+++ b/src/hotspot/share/opto/library_call.cpp
@@ -6580,6 +6580,10 @@ bool LibraryCallKit::inline_vectorizedMismatch() {
memory_phi = _gvn.transform(memory_phi);
result_phi = _gvn.transform(result_phi);
+ record_for_igvn(exit_block);
+ record_for_igvn(memory_phi);
+ record_for_igvn(result_phi);
+
set_control(exit_block);
set_all_memory(memory_phi);
set_result(result_phi);
diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp
index e515d2cfef4..edcb1b7bfad 100644
--- a/src/hotspot/share/opto/loopnode.cpp
+++ b/src/hotspot/share/opto/loopnode.cpp
@@ -458,7 +458,7 @@ Node* PhaseIdealLoop::loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Nod
return incr;
}
-Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xphi) {
+Node* PhaseIdealLoop::loop_iv_stride(Node* incr, Node*& xphi) {
assert(incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL, "caller resp.");
// Get merge point
xphi = incr->in(1);
@@ -474,7 +474,7 @@ Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xph
return stride;
}
-PhiNode* PhaseIdealLoop::loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop) {
+PhiNode* PhaseIdealLoop::loop_iv_phi(Node* xphi, Node* phi_incr, Node* x) {
if (!xphi->is_Phi()) {
return nullptr; // Too much math on the trip counter
}
@@ -1481,11 +1481,11 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi
assert(incr != nullptr && incr->Opcode() == Op_Add(bt), "no incr");
Node* xphi = nullptr;
- Node* stride = loop_iv_stride(incr, loop, xphi);
+ Node* stride = loop_iv_stride(incr, xphi);
assert(stride != nullptr, "no stride");
- PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop);
+ PhiNode* phi = loop_iv_phi(xphi, phi_incr, x);
assert(phi != nullptr && phi->in(LoopNode::LoopBackControl) == incr, "No phi");
@@ -1650,7 +1650,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
assert(incr->Opcode() == Op_Add(iv_bt), "wrong increment code");
Node* xphi = nullptr;
- Node* stride = loop_iv_stride(incr, loop, xphi);
+ Node* stride = loop_iv_stride(incr, xphi);
if (stride == nullptr) {
return false;
@@ -1664,7 +1664,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
jlong stride_con = stride->get_integer_as_long(iv_bt);
assert(stride_con != 0, "missed some peephole opt");
- PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop);
+ PhiNode* phi = loop_iv_phi(xphi, phi_incr, x);
if (phi == nullptr ||
(trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) ||
diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp
index cb5468bfd02..5fceb21ebd7 100644
--- a/src/hotspot/share/opto/loopnode.hpp
+++ b/src/hotspot/share/opto/loopnode.hpp
@@ -1285,8 +1285,8 @@ public:
Node* loop_exit_control(Node* x, IdealLoopTree* loop);
Node* loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob);
Node* loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr);
- Node* loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xphi);
- PhiNode* loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop);
+ Node* loop_iv_stride(Node* incr, Node*& xphi);
+ PhiNode* loop_iv_phi(Node* xphi, Node* phi_incr, Node* x);
bool is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt);
diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp
index 070749cfe1f..ddeb5f84d49 100644
--- a/src/hotspot/share/opto/loopopts.cpp
+++ b/src/hotspot/share/opto/loopopts.cpp
@@ -1676,6 +1676,10 @@ bool PhaseIdealLoop::safe_for_if_replacement(const Node* dom) const {
// like various versions of induction variable+offset. Clone the
// computation per usage to allow it to sink out of the loop.
void PhaseIdealLoop::try_sink_out_of_loop(Node* n) {
+ bool is_raw_to_oop_cast = n->is_ConstraintCast() &&
+ n->in(1)->bottom_type()->isa_rawptr() &&
+ !n->bottom_type()->isa_rawptr();
+
if (has_ctrl(n) &&
!n->is_Phi() &&
!n->is_Bool() &&
@@ -1685,7 +1689,9 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) {
!n->is_OpaqueNotNull() &&
!n->is_OpaqueInitializedAssertionPredicate() &&
!n->is_OpaqueTemplateAssertionPredicate() &&
- !n->is_Type()) {
+ !is_raw_to_oop_cast && // don't extend live ranges of raw oops
+ (KillPathsReachableByDeadTypeNode || !n->is_Type())
+ ) {
Node *n_ctrl = get_ctrl(n);
IdealLoopTree *n_loop = get_loop(n_ctrl);
@@ -4271,13 +4277,13 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old
}
assert(in->Opcode() == Op_AddI, "wrong increment code");
Node* xphi = nullptr;
- Node* stride = loop_iv_stride(in, loop, xphi);
+ Node* stride = loop_iv_stride(in, xphi);
if (stride == nullptr) {
continue;
}
- PhiNode* phi = loop_iv_phi(xphi, nullptr, head, loop);
+ PhiNode* phi = loop_iv_phi(xphi, nullptr, head);
if (phi == nullptr ||
(trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) ||
(trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != trunc1)) {
diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp
index 5fd262d54e1..3594806b91e 100644
--- a/src/hotspot/share/opto/machnode.hpp
+++ b/src/hotspot/share/opto/machnode.hpp
@@ -386,6 +386,13 @@ public:
// Returns true if this node is a check that can be implemented with a trap.
virtual bool is_TrapBasedCheckNode() const { return false; }
+
+ // Whether this node is expanded during code emission into a sequence of
+ // instructions and the first instruction can perform an implicit null check.
+ virtual bool is_late_expanded_null_check_candidate() const {
+ return false;
+ }
+
void set_removed() { add_flag(Flag_is_removed_by_peephole); }
bool get_removed() { return (flags() & Flag_is_removed_by_peephole) != 0; }
diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp
index c6ed2411fe3..8f21ee13e79 100644
--- a/src/hotspot/share/opto/macro.cpp
+++ b/src/hotspot/share/opto/macro.cpp
@@ -2407,7 +2407,6 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
}
}
// Next, attempt to eliminate allocations
- _has_locks = false;
progress = true;
while (progress) {
progress = false;
@@ -2431,7 +2430,6 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
case Node::Class_Lock:
case Node::Class_Unlock:
assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
- _has_locks = true;
break;
case Node::Class_ArrayCopy:
break;
diff --git a/src/hotspot/share/opto/macro.hpp b/src/hotspot/share/opto/macro.hpp
index 4654789a290..7f27688a57a 100644
--- a/src/hotspot/share/opto/macro.hpp
+++ b/src/hotspot/share/opto/macro.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -83,9 +83,6 @@ private:
// projections extracted from a call node
CallProjections _callprojs;
- // Additional data collected during macro expansion
- bool _has_locks;
-
void expand_allocate(AllocateNode *alloc);
void expand_allocate_array(AllocateArrayNode *alloc);
void expand_allocate_common(AllocateNode* alloc,
@@ -199,7 +196,7 @@ private:
Node* make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, Node* ctl, Node* mem, BasicType ft, const Type *ftype, AllocateNode *alloc);
public:
- PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn), _has_locks(false) {
+ PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn) {
_igvn.set_delay_transform(true);
}
diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp
index 930c355fa62..f0a7b742998 100644
--- a/src/hotspot/share/opto/output.cpp
+++ b/src/hotspot/share/opto/output.cpp
@@ -2015,8 +2015,10 @@ void PhaseOutput::FillExceptionTables(uint cnt, uint *call_returns, uint *inct_s
// Handle implicit null exception table updates
if (n->is_MachNullCheck()) {
- assert(n->in(1)->as_Mach()->barrier_data() == 0,
- "Implicit null checks on memory accesses with barriers are not yet supported");
+ MachNode* access = n->in(1)->as_Mach();
+ assert(access->barrier_data() == 0 ||
+ access->is_late_expanded_null_check_candidate(),
+ "Implicit null checks on memory accesses with barriers are only supported on nodes explicitly marked as null-check candidates");
uint block_num = block->non_connector_successor(0)->_pre_order;
_inc_table.append(inct_starts[inct_cnt++], blk_labels[block_num].loc_pos());
continue;
diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp
index 953afb0e830..1ba8e145e7d 100644
--- a/src/hotspot/share/opto/phaseX.cpp
+++ b/src/hotspot/share/opto/phaseX.cpp
@@ -1072,7 +1072,11 @@ void PhaseIterGVN::optimize() {
#ifdef ASSERT
void PhaseIterGVN::verify_optimize() {
- if (is_verify_Value()) {
+ assert(_worklist.size() == 0, "igvn worklist must be empty before verify");
+
+ if (is_verify_Value() ||
+ is_verify_Ideal() ||
+ is_verify_Identity()) {
ResourceMark rm;
Unique_Node_List worklist;
bool failure = false;
@@ -1080,7 +1084,10 @@ void PhaseIterGVN::verify_optimize() {
worklist.push(C->root());
for (uint j = 0; j < worklist.size(); ++j) {
Node* n = worklist.at(j);
- failure |= verify_node_value(n);
+ if (is_verify_Value()) { failure |= verify_Value_for(n); }
+ if (is_verify_Ideal()) { failure |= verify_Ideal_for(n, false); }
+ if (is_verify_Ideal()) { failure |= verify_Ideal_for(n, true); }
+ if (is_verify_Identity()) { failure |= verify_Identity_for(n); }
// traverse all inputs and outputs
for (uint i = 0; i < n->req(); i++) {
if (n->in(i) != nullptr) {
@@ -1097,6 +1104,27 @@ void PhaseIterGVN::verify_optimize() {
// in the verification code above if that is not possible for some reason (like Load nodes).
assert(!failure, "Missed optimization opportunity in PhaseIterGVN");
}
+
+ verify_empty_worklist(nullptr);
+}
+
+void PhaseIterGVN::verify_empty_worklist(Node* node) {
+ // Verify that the igvn worklist is empty. If no optimization happened, then
+ // nothing needs to be on the worklist.
+ if (_worklist.size() == 0) { return; }
+
+ stringStream ss; // Print as a block without tty lock.
+ for (uint j = 0; j < _worklist.size(); j++) {
+ Node* n = _worklist.at(j);
+ ss.print("igvn.worklist[%d] ", j);
+ n->dump("\n", false, &ss);
+ }
+ if (_worklist.size() != 0 && node != nullptr) {
+ ss.print_cr("Previously optimized:");
+ node->dump("\n", false, &ss);
+ }
+ tty->print_cr("%s", ss.as_string());
+ assert(false, "igvn worklist must still be empty after verify");
}
// Check that type(n) == n->Value(), return true if we have a failure.
@@ -1104,7 +1132,7 @@ void PhaseIterGVN::verify_optimize() {
// (1) Integer "widen" changes, but the range is the same.
// (2) LoadNode performs deep traversals. Load is not notified for changes far away.
// (3) CmpPNode performs deep traversals if it compares oopptr. CmpP is not notified for changes far away.
-bool PhaseIterGVN::verify_node_value(Node* n) {
+bool PhaseIterGVN::verify_Value_for(Node* n) {
// If we assert inside type(n), because the type is still a null, then maybe
// the node never went through gvn.transform, which would be a bug.
const Type* told = type(n);
@@ -1149,15 +1177,873 @@ bool PhaseIterGVN::verify_node_value(Node* n) {
// after loop-opts, so that should take care of many of these cases.
return false;
}
- tty->cr();
- tty->print_cr("Missed Value optimization:");
- n->dump_bfs(1, nullptr, "");
- tty->print_cr("Current type:");
- told->dump_on(tty);
- tty->cr();
- tty->print_cr("Optimized type:");
- tnew->dump_on(tty);
- tty->cr();
+
+ stringStream ss; // Print as a block without tty lock.
+ ss.cr();
+ ss.print_cr("Missed Value optimization:");
+ n->dump_bfs(1, nullptr, "", &ss);
+ ss.print_cr("Current type:");
+ told->dump_on(&ss);
+ ss.cr();
+ ss.print_cr("Optimized type:");
+ tnew->dump_on(&ss);
+ ss.cr();
+ tty->print_cr("%s", ss.as_string());
+ return true;
+}
+
+// Check that all Ideal optimizations that could be done were done.
+// Returns true if it found missed optimization opportunities and
+// false otherwise (no missed optimization, or skipped verification).
+bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
+ // First, we check a list of exceptions, where we skip verification,
+ // because there are known cases where Ideal can optimize after IGVN.
+ // Some may be expected and cannot be fixed, and others should be fixed.
+ switch (n->Opcode()) {
+ // RangeCheckNode::Ideal looks up the chain for about 999 nodes
+ // (see "Range-Check scan limit"). So, it is possible that something
+ // is optimized in that input subgraph, and the RangeCheck was not
+ // added to the worklist because it would be too expensive to walk
+ // down the graph for 1000 nodes and put all on the worklist.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xbatch --version
+ case Op_RangeCheck:
+ return false;
+
+ // IfNode::Ideal does:
+ // Node* prev_dom = search_identical(dist, igvn);
+ // which means we seach up the CFG, traversing at most up to a distance.
+ // If anything happens rather far away from the If, we may not put the If
+ // back on the worklist.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_If:
+ return false;
+
+ // IfNode::simple_subsuming
+ // Looks for dominating test that subsumes the current test.
+ // Notification could be difficult because of larger distance.
+ //
+ // Found with:
+ // runtime/exceptionMsgs/ArrayIndexOutOfBoundsException/ArrayIndexOutOfBoundsExceptionTest.java#id1
+ // -XX:VerifyIterativeGVN=1110
+ case Op_CountedLoopEnd:
+ return false;
+
+ // LongCountedLoopEndNode::Ideal
+ // Probably same issue as above.
+ //
+ // Found with:
+ // compiler/predicates/assertion/TestAssertionPredicates.java#NoLoopPredicationXbatch
+ // -XX:StressLongCountedLoop=2000000 -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ case Op_LongCountedLoopEnd:
+ return false;
+
+ // RegionNode::Ideal does "Skip around the useless IF diamond".
+ // 245 IfTrue === 244
+ // 258 If === 245 257
+ // 259 IfTrue === 258 [[ 263 ]]
+ // 260 IfFalse === 258 [[ 263 ]]
+ // 263 Region === 263 260 259 [[ 263 268 ]]
+ // to
+ // 245 IfTrue === 244
+ // 263 Region === 263 245 _ [[ 263 268 ]]
+ //
+ // "Useless" means that there is no code in either branch of the If.
+ // I found a case where this was not done yet during IGVN.
+ // Why does the Region not get added to IGVN worklist when the If diamond becomes useless?
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_Region:
+ return false;
+
+ // In AddNode::Ideal, we call "commute", which swaps the inputs so
+ // that smaller idx are first. Tracking it back, it led me to
+ // PhaseIdealLoop::remix_address_expressions which swapped the edges.
+ //
+ // Example:
+ // Before PhaseIdealLoop::remix_address_expressions
+ // 154 AddI === _ 12 144
+ // After PhaseIdealLoop::remix_address_expressions
+ // 154 AddI === _ 144 12
+ // After AddNode::Ideal
+ // 154 AddI === _ 12 144
+ //
+ // I suspect that the node should be added to the IGVN worklist after
+ // PhaseIdealLoop::remix_address_expressions
+ //
+ // This is the only case I looked at, there may be others. Found like this:
+ // java -XX:VerifyIterativeGVN=0100 -Xbatch --version
+ //
+ // The following hit the same logic in PhaseIdealLoop::remix_address_expressions.
+ //
+ // Note: currently all of these fail also for other reasons, for example
+ // because of "commute" doing the reordering with the phi below. Once
+ // that is resolved, we can come back to this issue here.
+ //
+ // case Op_AddD:
+ // case Op_AddI:
+ // case Op_AddL:
+ // case Op_AddF:
+ // case Op_MulI:
+ // case Op_MulL:
+ // case Op_MulF:
+ // case Op_MulD:
+ // if (n->in(1)->_idx > n->in(2)->_idx) {
+ // // Expect "commute" to revert this case.
+ // return false;
+ // }
+ // break; // keep verifying
+
+ // AddFNode::Ideal calls "commute", which can reorder the inputs for this:
+ // Check for tight loop increments: Loop-phi of Add of loop-phi
+ // It wants to take the phi into in(1):
+ // 471 Phi === 435 38 390
+ // 390 AddF === _ 471 391
+ //
+ // Other Associative operators are also affected equally.
+ //
+ // Investigate why this does not happen earlier during IGVN.
+ //
+ // Found with:
+ // test/hotspot/jtreg/compiler/loopopts/superword/ReductionPerf.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_AddD:
+ //case Op_AddI: // Also affected for other reasons, see case further down.
+ //case Op_AddL: // Also affected for other reasons, see case further down.
+ case Op_AddF:
+ case Op_MulI:
+ case Op_MulL:
+ case Op_MulF:
+ case Op_MulD:
+ case Op_MinF:
+ case Op_MinD:
+ case Op_MaxF:
+ case Op_MaxD:
+ // XorINode::Ideal
+ // Found with:
+ // compiler/intrinsics/chacha/TestChaCha20.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_XorI:
+ case Op_XorL:
+ // It seems we may have similar issues with the HF cases.
+ // Found with aarch64:
+ // compiler/vectorization/TestFloat16VectorOperations.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_AddHF:
+ case Op_MulHF:
+ case Op_MaxHF:
+ case Op_MinHF:
+ return false;
+
+ // In MulNode::Ideal the edges can be swapped to help value numbering:
+ //
+ // // We are OK if right is a constant, or right is a load and
+ // // left is a non-constant.
+ // if( !(t2->singleton() ||
+ // (in(2)->is_Load() && !(t1->singleton() || in(1)->is_Load())) ) ) {
+ // if( t1->singleton() || // Left input is a constant?
+ // // Otherwise, sort inputs (commutativity) to help value numbering.
+ // (in(1)->_idx > in(2)->_idx) ) {
+ // swap_edges(1, 2);
+ //
+ // Why was this not done earlier during IGVN?
+ //
+ // Found with:
+ // test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithG1.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_AndI:
+ // Same for AndL.
+ // Found with:
+ // compiler/intrinsics/bigInteger/MontgomeryMultiplyTest.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_AndL:
+ return false;
+
+ // SubLNode::Ideal does transform like:
+ // Convert "c1 - (y+c0)" into "(c1-c0) - y"
+ //
+ // In IGVN before verification:
+ // 8423 ConvI2L === _ 3519 [[ 8424 ]] #long:-2
+ // 8422 ConvI2L === _ 8399 [[ 8424 ]] #long:3..256:www
+ // 8424 AddL === _ 8422 8423 [[ 8383 ]] !orig=[8382]
+ // 8016 ConL === 0 [[ 8383 ]] #long:0
+ // 8383 SubL === _ 8016 8424 [[ 8156 ]] !orig=[8154]
+ //
+ // And then in verification:
+ // 8338 ConL === 0 [[ 8339 8424 ]] #long:-2 <----- Was constant folded.
+ // 8422 ConvI2L === _ 8399 [[ 8424 ]] #long:3..256:www
+ // 8424 AddL === _ 8422 8338 [[ 8383 ]] !orig=[8382]
+ // 8016 ConL === 0 [[ 8383 ]] #long:0
+ // 8383 SubL === _ 8016 8424 [[ 8156 ]] !orig=[8154]
+ //
+ // So the form changed from:
+ // c1 - (y + [8423 ConvI2L])
+ // to
+ // c1 - (y + -2)
+ // but the SubL was not added to the IGVN worklist. Investigate why.
+ // There could be other issues too.
+ //
+ // There seems to be a related AddL IGVN optimization that triggers
+ // the same SubL optimization, so investigate that too.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_SubL:
+ return false;
+
+ // SubINode::Ideal does
+ // Convert "x - (y+c0)" into "(x-y) - c0" AND
+ // Convert "c1 - (y+c0)" into "(c1-c0) - y"
+ //
+ // Investigate why this does not yet happen during IGVN.
+ //
+ // Found with:
+ // test/hotspot/jtreg/compiler/c2/IVTest.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_SubI:
+ return false;
+
+ // AddNode::IdealIL does transform like:
+ // Convert x + (con - y) into "(x - y) + con"
+ //
+ // In IGVN before verification:
+ // 8382 ConvI2L
+ // 8381 ConvI2L === _ 791 [[ 8383 ]] #long:0
+ // 8383 SubL === _ 8381 8382
+ // 8168 ConvI2L
+ // 8156 AddL === _ 8168 8383 [[ 8158 ]]
+ //
+ // And then in verification:
+ // 8424 AddL
+ // 8016 ConL === 0 [[ 8383 ]] #long:0 <--- Was constant folded.
+ // 8383 SubL === _ 8016 8424
+ // 8168 ConvI2L
+ // 8156 AddL === _ 8168 8383 [[ 8158 ]]
+ //
+ // So the form changed from:
+ // x + (ConvI2L(0) - [8382 ConvI2L])
+ // to
+ // x + (0 - [8424 AddL])
+ // but the AddL was not added to the IGVN worklist. Investigate why.
+ // There could be other issues, too. For example with "commute", see above.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_AddL:
+ return false;
+
+ // SubTypeCheckNode::Ideal calls SubTypeCheckNode::verify_helper, which does
+ // Node* cmp = phase->transform(new CmpPNode(subklass, in(SuperKlass)));
+ // record_for_cleanup(cmp, phase);
+ // This verification code in the Ideal code creates new nodes, and checks
+ // if they fold in unexpected ways. This means some nodes are created and
+ // added to the worklist, even if the SubTypeCheck is not optimized. This
+ // goes agains the assumption of the verification here, which assumes that
+ // if the node is not optimized, then no new nodes should be created, and
+ // also no nodes should be added to the worklist.
+ // I see two options:
+ // 1) forbid what verify_helper does, because for each Ideal call it
+ // uses memory and that is suboptimal. But it is not clear how that
+ // verification can be done otherwise.
+ // 2) Special case the verification here. Probably the new nodes that
+ // were just created are dead, i.e. they are not connected down to
+ // root. We could verify that, and remove those nodes from the graph
+ // by setting all their inputs to nullptr. And of course we would
+ // have to remove those nodes from the worklist.
+ // Maybe there are other options too, I did not dig much deeper yet.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xbatch --version
+ case Op_SubTypeCheck:
+ return false;
+
+ // LoopLimitNode::Ideal when stride is constant power-of-2, we can do a lowering
+ // to other nodes: Conv, Add, Sub, Mul, And ...
+ //
+ // 107 ConI === 0 [[ ... ]] #int:2
+ // 84 LoadRange === _ 7 83
+ // 50 ConI === 0 [[ ... ]] #int:0
+ // 549 LoopLimit === _ 50 84 107
+ //
+ // I stepped backward, to see how the node was generated, and I found that it was
+ // created in PhaseIdealLoop::exact_limit and not changed since. It is added to the
+ // IGVN worklist. I quickly checked when it goes into LoopLimitNode::Ideal after
+ // that, and it seems we want to skip lowering it until after loop-opts, but never
+ // add call record_for_post_loop_opts_igvn. This would be an easy fix, but there
+ // could be other issues too.
+ //
+ // Fond with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_LoopLimit:
+ return false;
+
+ // PhiNode::Ideal calls split_flow_path, which tries to do this:
+ // "This optimization tries to find two or more inputs of phi with the same constant
+ // value. It then splits them into a separate Phi, and according Region."
+ //
+ // Example:
+ // 130 DecodeN === _ 129
+ // 50 ConP === 0 [[ 18 91 99 18 ]] #null
+ // 18 Phi === 14 50 130 50 [[ 133 ]] #java/lang/Object * Oop:java/lang/Object *
+ //
+ // turns into:
+ //
+ // 50 ConP === 0 [[ 99 91 18 ]] #null
+ // 130 DecodeN === _ 129 [[ 18 ]]
+ // 18 Phi === 14 130 50 [[ 133 ]] #java/lang/Object * Oop:java/lang/Object *
+ //
+ // We would have to investigate why this optimization does not happen during IGVN.
+ // There could also be other issues - I did not investigate further yet.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_Phi:
+ return false;
+
+ // MemBarNode::Ideal does "Eliminate volatile MemBars for scalar replaced objects".
+ // For examle "The allocated object does not escape".
+ //
+ // It seems the difference to earlier calls to MemBarNode::Ideal, is that there
+ // alloc->as_Allocate()->does_not_escape_thread() returned false, but in verification
+ // it returned true. Why does the MemBarStoreStore not get added to the IGVN
+ // worklist when this change happens?
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_MemBarStoreStore:
+ return false;
+
+ // ConvI2LNode::Ideal converts
+ // 648 AddI === _ 583 645 [[ 661 ]]
+ // 661 ConvI2L === _ 648 [[ 664 ]] #long:0..maxint-1:www
+ // into
+ // 772 ConvI2L === _ 645 [[ 773 ]] #long:-120..maxint-61:www
+ // 771 ConvI2L === _ 583 [[ 773 ]] #long:60..120:www
+ // 773 AddL === _ 771 772 [[ ]]
+ //
+ // We have to investigate why this does not happen during IGVN in this case.
+ // There could also be other issues - I did not investigate further yet.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ case Op_ConvI2L:
+ return false;
+
+ // AddNode::IdealIL can do this transform (and similar other ones):
+ // Convert "a*b+a*c into a*(b+c)
+ // The example had AddI(MulI(a, b), MulI(a, c)). Why did this not happen
+ // during IGVN? There was a mutation for one of the MulI, and only
+ // after that the pattern was as needed for the optimization. The MulI
+ // was added to the IGVN worklist, but not the AddI. This probably
+ // can be fixed by adding the correct pattern in add_users_of_use_to_worklist.
+ //
+ // Found with:
+ // test/hotspot/jtreg/compiler/loopopts/superword/ReductionPerf.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_AddI:
+ return false;
+
+ // ArrayCopyNode::Ideal
+ // calls ArrayCopyNode::prepare_array_copy
+ // calls Compile::conv_I2X_index -> is called with sizetype = intcon(0), I think that
+ // is not expected, and we create a range int:0..-1
+ // calls Compile::constrained_convI2L -> creates ConvI2L(intcon(1), int:0..-1)
+ // note: the type is already empty!
+ // calls PhaseIterGVN::transform
+ // calls PhaseIterGVN::transform_old
+ // calls PhaseIterGVN::subsume_node -> subsume ConvI2L with TOP
+ // calls Unique_Node_List::push -> pushes TOP to worklist
+ //
+ // Once we get back to ArrayCopyNode::prepare_array_copy, we get back TOP, and
+ // return false. This means we eventually return nullptr from ArrayCopyNode::Ideal.
+ //
+ // Question: is it ok to push anything to the worklist during ::Ideal, if we will
+ // return nullptr, indicating nothing happened?
+ // Is it smart to do transform in Compile::constrained_convI2L, and then
+ // check for TOP in calls ArrayCopyNode::prepare_array_copy?
+ // Should we just allow TOP to land on the worklist, as an exception?
+ //
+ // Found with:
+ // compiler/arraycopy/TestArrayCopyAsLoadsStores.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_ArrayCopy:
+ return false;
+
+ // CastLLNode::Ideal
+ // calls ConstraintCastNode::optimize_integer_cast -> pushes CastLL through SubL
+ //
+ // Could be a notification issue, where updates inputs of CastLL do not notify
+ // down through SubL to CastLL.
+ //
+ // Found With:
+ // compiler/c2/TestMergeStoresMemorySegment.java#byte-array
+ // -XX:VerifyIterativeGVN=1110
+ case Op_CastLL:
+ return false;
+
+ // Similar case happens to CastII
+ //
+ // Found With:
+ // compiler/c2/TestScalarReplacementMaxLiveNodes.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_CastII:
+ return false;
+
+ // MaxLNode::Ideal
+ // calls AddNode::Ideal
+ // calls commute -> decides to swap edges
+ //
+ // Another notification issue, because we check inputs of inputs?
+ // MaxL -> Phi -> Loop
+ // MaxL -> Phi -> MaxL
+ //
+ // Found with:
+ // compiler/c2/irTests/TestIfMinMax.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_MaxL:
+ case Op_MinL:
+ return false;
+
+ // OrINode::Ideal
+ // calls AddNode::Ideal
+ // calls commute -> left is Load, right not -> commute.
+ //
+ // Not sure why notification does not work here, seems like
+ // the depth is only 1, so it should work. Needs investigation.
+ //
+ // Found with:
+ // compiler/codegen/TestCharVect2.java#id0
+ // -XX:VerifyIterativeGVN=1110
+ case Op_OrI:
+ case Op_OrL:
+ return false;
+
+ // Bool -> constant folded to 1.
+ // Issue with notification?
+ //
+ // Found with:
+ // compiler/c2/irTests/TestVectorizationMismatchedAccess.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_Bool:
+ return false;
+
+ // LShiftLNode::Ideal
+ // Looks at pattern: "(x + x) << c0", converts it to "x << (c0 + 1)"
+ // Probably a notification issue.
+ //
+ // Found with:
+ // compiler/conversions/TestMoveConvI2LOrCastIIThruAddIs.java
+ // -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ case Op_LShiftL:
+ return false;
+
+ // LShiftINode::Ideal
+ // pattern: ((x + con1) << con2) -> x << con2 + con1 << con2
+ // Could be issue with notification of inputs of inputs
+ //
+ // Side-note: should cases like these not be shared between
+ // LShiftI and LShiftL?
+ //
+ // Found with:
+ // compiler/escapeAnalysis/Test6689060.java
+ // -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110 -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ case Op_LShiftI:
+ return false;
+
+ // AddPNode::Ideal seems to do set_req without removing lock first.
+ // Found with various vector tests tier1-tier3.
+ case Op_AddP:
+ return false;
+
+ // StrIndexOfNode::Ideal
+ // Found in tier1-3.
+ case Op_StrIndexOf:
+ case Op_StrIndexOfChar:
+ return false;
+
+ // StrEqualsNode::Identity
+ //
+ // Found (linux x64 only?) with:
+ // serviceability/sa/ClhsdbThreadContext.java
+ // -XX:+UnlockExperimentalVMOptions -XX:LockingMode=1 -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ case Op_StrEquals:
+ return false;
+
+ // AryEqNode::Ideal
+ // Not investigated. Reshapes itself and adds lots of nodes to the worklist.
+ //
+ // Found with:
+ // vmTestbase/vm/mlvm/meth/stress/compiler/i2c_c2i/Test.java
+ // -XX:+UnlockDiagnosticVMOptions -XX:-TieredCompilation -XX:+StressUnstableIfTraps -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ case Op_AryEq:
+ return false;
+
+ // MergeMemNode::Ideal
+ // Found in tier1-3. Did not investigate further yet.
+ case Op_MergeMem:
+ return false;
+
+ // URShiftINode::Ideal
+ // Found in tier1-3. Did not investigate further yet.
+ case Op_URShiftI:
+ return false;
+
+ // CMoveINode::Ideal
+ // Found in tier1-3. Did not investigate further yet.
+ case Op_CMoveI:
+ return false;
+
+ // CmpPNode::Ideal calls isa_const_java_mirror
+ // and generates new constant nodes, even if no progress is made.
+ // We can probably rewrite this so that only types are generated.
+ // It seems that object types are not hashed, we could investigate
+ // if that is an option as well.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=1110 -Xcomp --version
+ case Op_CmpP:
+ return false;
+
+ // MinINode::Ideal
+ // Did not investigate, but there are some patterns that might
+ // need more notification.
+ case Op_MinI:
+ case Op_MaxI: // preemptively removed it as well.
+ return false;
+ }
+
+ if (n->is_Load()) {
+ // LoadNode::Ideal uses tries to find an earlier memory state, and
+ // checks can_see_stored_value for it.
+ //
+ // Investigate why this was not already done during IGVN.
+ // A similar issue happens with Identity.
+ //
+ // There seem to be other cases where loads go up some steps, like
+ // LoadNode::Ideal going up 10x steps to find dominating load.
+ //
+ // Found with:
+ // test/hotspot/jtreg/compiler/arraycopy/TestCloneAccess.java
+ // -XX:VerifyIterativeGVN=1110
+ return false;
+ }
+
+ if (n->is_Store()) {
+ // StoreNode::Ideal can do this:
+ // // Capture an unaliased, unconditional, simple store into an initializer.
+ // // Or, if it is independent of the allocation, hoist it above the allocation.
+ // That replaces the Store with a MergeMem.
+ //
+ // We have to investigate why this does not happen during IGVN in this case.
+ // There could also be other issues - I did not investigate further yet.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=0100 -Xcomp --version
+ return false;
+ }
+
+ if (n->is_Vector()) {
+ // VectorNode::Ideal swaps edges, but only for ops
+ // that are deemed commutable. But swap_edges
+ // requires the hash to be invariant when the edges
+ // are swapped, which is not implemented for these
+ // vector nodes. This seems not to create any trouble
+ // usually, but we can also get graphs where in the
+ // end the nodes are not all commuted, so there is
+ // definitively an issue here.
+ //
+ // Probably we have two options: kill the hash, or
+ // properly make the hash commutation friendly.
+ //
+ // Found with:
+ // compiler/vectorapi/TestMaskedMacroLogicVector.java
+ // -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110 -XX:+UseParallelGC -XX:+UseNUMA
+ return false;
+ }
+
+ if (n->is_Region()) {
+ // LoopNode::Ideal calls RegionNode::Ideal.
+ // CountedLoopNode::Ideal calls RegionNode::Ideal too.
+ // But I got an issue because RegionNode::optimize_trichotomy
+ // then modifies another node, and pushes nodes to the worklist
+ // Not sure if this is ok, modifying another node like that.
+ // Maybe it is, then we need to look into what to do with
+ // the nodes that are now on the worklist, maybe just clear
+ // them out again. But maybe modifying other nodes like that
+ // is also bad design. In the end, we return nullptr for
+ // the current CountedLoop. But the extra nodes on the worklist
+ // trip the asserts later on.
+ //
+ // Found with:
+ // compiler/eliminateAutobox/TestShortBoxing.java
+ // -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ return false;
+ }
+
+ if (n->is_CallJava()) {
+ // CallStaticJavaNode::Ideal
+ // Led to a crash:
+ // assert((is_CallStaticJava() && cg->is_mh_late_inline()) || (is_CallDynamicJava() && cg->is_virtual_late_inline())) failed: mismatch
+ //
+ // Did not investigate yet, could be a bug.
+ // Or maybe it does not expect to be called during verification.
+ //
+ // Found with:
+ // test/jdk/jdk/incubator/vector/VectorRuns.java
+ // -XX:VerifyIterativeGVN=1110
+
+ // CallDynamicJavaNode::Ideal, and I think also for CallStaticJavaNode::Ideal
+ // and possibly their subclasses.
+ // During late inlining it can call CallJavaNode::register_for_late_inline
+ // That means we do more rounds of late inlining, but might fail.
+ // Then we do IGVN again, and register the node again for late inlining.
+ // This creates an endless cycle. Everytime we try late inlining, we
+ // are also creating more nodes, especially SafePoint and MergeMem.
+ // These nodes are immediately rejected when the inlining fails in the
+ // do_late_inline_check, but they still grow the memory, until we hit
+ // the MemLimit and crash.
+ // The assumption here seems that CallDynamicJavaNode::Ideal does not get
+ // called repeatedly, and eventually we terminate. I fear this is not
+ // a great assumption to make. We should investigate more.
+ //
+ // Found with:
+ // compiler/loopopts/superword/TestDependencyOffsets.java#vanilla-U
+ // -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ return false;
+ }
+
+ // The number of nodes shoud not increase.
+ uint old_unique = C->unique();
+
+ Node* i = n->Ideal(this, can_reshape);
+ // If there was no new Idealization, we are probably happy.
+ if (i == nullptr) {
+ if (old_unique < C->unique()) {
+ stringStream ss; // Print as a block without tty lock.
+ ss.cr();
+ ss.print_cr("Ideal optimization did not make progress but created new unused nodes.");
+ ss.print_cr(" old_unique = %d, unique = %d", old_unique, C->unique());
+ n->dump_bfs(1, nullptr, "", &ss);
+ tty->print_cr("%s", ss.as_string());
+ return true;
+ }
+
+ verify_empty_worklist(n);
+
+ // Everything is good.
+ return false;
+ }
+
+ // We just saw a new Idealization which was not done during IGVN.
+ stringStream ss; // Print as a block without tty lock.
+ ss.cr();
+ ss.print_cr("Missed Ideal optimization (can_reshape=%s):", can_reshape ? "true": "false");
+ if (i == n) {
+ ss.print_cr("The node was reshaped by Ideal.");
+ } else {
+ ss.print_cr("The node was replaced by Ideal.");
+ ss.print_cr("Old node:");
+ n->dump_bfs(1, nullptr, "", &ss);
+ }
+ ss.print_cr("The result after Ideal:");
+ i->dump_bfs(1, nullptr, "", &ss);
+ tty->print_cr("%s", ss.as_string());
+ return true;
+}
+
+// Check that all Identity optimizations that could be done were done.
+// Returns true if it found missed optimization opportunities and
+// false otherwise (no missed optimization, or skipped verification).
+bool PhaseIterGVN::verify_Identity_for(Node* n) {
+ // First, we check a list of exceptions, where we skip verification,
+ // because there are known cases where Ideal can optimize after IGVN.
+ // Some may be expected and cannot be fixed, and others should be fixed.
+ switch (n->Opcode()) {
+ // SafePointNode::Identity can remove SafePoints, but wants to wait until
+ // after loopopts:
+ // // Transforming long counted loops requires a safepoint node. Do not
+ // // eliminate a safepoint until loop opts are over.
+ // if (in(0)->is_Proj() && !phase->C->major_progress()) {
+ //
+ // I think the check for major_progress does delay it until after loopopts
+ // but it does not ensure that the node is on the IGVN worklist after
+ // loopopts. I think we should try to instead check for
+ // phase->C->post_loop_opts_phase() and call record_for_post_loop_opts_igvn.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=1000 -Xcomp --version
+ case Op_SafePoint:
+ return false;
+
+ // MergeMemNode::Identity replaces the MergeMem with its base_memory if it
+ // does not record any other memory splits.
+ //
+ // I did not deeply investigate, but it looks like MergeMemNode::Identity
+ // never got called during IGVN for this node, investigate why.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=1000 -Xcomp --version
+ case Op_MergeMem:
+ return false;
+
+ // ConstraintCastNode::Identity finds casts that are the same, except that
+ // the control is "higher up", i.e. dominates. The call goes via
+ // ConstraintCastNode::dominating_cast to PhaseGVN::is_dominator_helper,
+ // which traverses up to 100 idom steps. If anything gets optimized somewhere
+ // away from the cast, but within 100 idom steps, the cast may not be
+ // put on the IGVN worklist any more.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=1000 -Xcomp --version
+ case Op_CastPP:
+ case Op_CastII:
+ case Op_CastLL:
+ return false;
+
+ // Same issue for CheckCastPP, uses ConstraintCastNode::Identity and
+ // checks dominator, which may be changed, but too far up for notification
+ // to work.
+ //
+ // Found with:
+ // compiler/c2/irTests/TestSkeletonPredicates.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_CheckCastPP:
+ return false;
+
+ // In SubNode::Identity, we do:
+ // Convert "(X+Y) - Y" into X and "(X+Y) - X" into Y
+ // In the example, the AddI had an input replaced, the AddI is
+ // added to the IGVN worklist, but the SubI is one link further
+ // down and is not added. I checked add_users_of_use_to_worklist
+ // where I would expect the SubI would be added, and I cannot
+ // find the pattern, only this one:
+ // If changed AddI/SubI inputs, check CmpU for range check optimization.
+ //
+ // Fix this "notification" issue and check if there are any other
+ // issues.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=1000 -Xcomp --version
+ case Op_SubI:
+ case Op_SubL:
+ return false;
+
+ // PhiNode::Identity checks for patterns like:
+ // r = (x != con) ? x : con;
+ // that can be constant folded to "x".
+ //
+ // Call goes through PhiNode::is_cmove_id and CMoveNode::is_cmove_id.
+ // I suspect there was some earlier change to one of the inputs, but
+ // not all relevant outputs were put on the IGVN worklist.
+ //
+ // Found with:
+ // test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithG1.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_Phi:
+ return false;
+
+ // ConvI2LNode::Identity does
+ // convert I2L(L2I(x)) => x
+ //
+ // Investigate why this did not already happen during IGVN.
+ //
+ // Found with:
+ // compiler/loopopts/superword/TestDependencyOffsets.java#vanilla-A
+ // -XX:VerifyIterativeGVN=1110
+ case Op_ConvI2L:
+ return false;
+
+ // MaxNode::find_identity_operation
+ // Finds patterns like Max(A, Max(A, B)) -> Max(A, B)
+ // This can be a 2-hop search, so maybe notification is not
+ // good enough.
+ //
+ // Found with:
+ // compiler/codegen/TestBooleanVect.java
+ // -XX:VerifyIterativeGVN=1110
+ case Op_MaxL:
+ case Op_MinL:
+ case Op_MaxI:
+ case Op_MinI:
+ case Op_MaxF:
+ case Op_MinF:
+ case Op_MaxHF:
+ case Op_MinHF:
+ case Op_MaxD:
+ case Op_MinD:
+ return false;
+
+
+ // AddINode::Identity
+ // Converts (x-y)+y to x
+ // Could be issue with notification
+ //
+ // Turns out AddL does the same.
+ //
+ // Found with:
+ // compiler/c2/Test6792161.java
+ // -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ case Op_AddI:
+ case Op_AddL:
+ return false;
+
+ // AbsINode::Identity
+ // Not investigated yet.
+ case Op_AbsI:
+ return false;
+ }
+
+ if (n->is_Load()) {
+ // LoadNode::Identity tries to look for an earlier store value via
+ // can_see_stored_value. I found an example where this led to
+ // an Allocation, where we could assume the value was still zero.
+ // So the LoadN can be replaced with a zerocon.
+ //
+ // Investigate why this was not already done during IGVN.
+ // A similar issue happens with Ideal.
+ //
+ // Found with:
+ // java -XX:VerifyIterativeGVN=1000 -Xcomp --version
+ return false;
+ }
+
+ if (n->is_Store()) {
+ // StoreNode::Identity
+ // Not investigated, but found missing optimization for StoreI.
+ // Looks like a StoreI is replaced with an InitializeNode.
+ //
+ // Found with:
+ // applications/ctw/modules/java_base_2.java
+ // -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -Djava.awt.headless=true -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
+ return false;
+ }
+
+ if (n->is_Vector()) {
+ // Found with tier1-3. Not investigated yet.
+ // The observed issue was with AndVNode::Identity
+ return false;
+ }
+
+ Node* i = n->Identity(this);
+ // If we cannot find any other Identity, we are happy.
+ if (i == n) {
+ verify_empty_worklist(n);
+ return false;
+ }
+
+ // The verification just found a new Identity that was not found during IGVN.
+ stringStream ss; // Print as a block without tty lock.
+ ss.cr();
+ ss.print_cr("Missed Identity optimization:");
+ ss.print_cr("Old node:");
+ n->dump_bfs(1, nullptr, "", &ss);
+ ss.print_cr("New node:");
+ i->dump_bfs(1, nullptr, "", &ss);
+ tty->print_cr("%s", ss.as_string());
return true;
}
#endif
@@ -1890,12 +2776,12 @@ void PhaseCCP::analyze() {
#ifdef ASSERT
// For every node n on verify list, check if type(n) == n->Value()
-// We have a list of exceptions, see comments in verify_node_value.
+// We have a list of exceptions, see comments in verify_Value_for.
void PhaseCCP::verify_analyze(Unique_Node_List& worklist_verify) {
bool failure = false;
while (worklist_verify.size()) {
Node* n = worklist_verify.pop();
- failure |= verify_node_value(n);
+ failure |= verify_Value_for(n);
}
// If we get this assert, check why the reported nodes were not processed again in CCP.
// We should either make sure that these nodes are properly added back to the CCP worklist
diff --git a/src/hotspot/share/opto/phaseX.hpp b/src/hotspot/share/opto/phaseX.hpp
index c2a0f0dbb77..648e911e783 100644
--- a/src/hotspot/share/opto/phaseX.hpp
+++ b/src/hotspot/share/opto/phaseX.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -494,7 +494,10 @@ public:
void optimize();
#ifdef ASSERT
void verify_optimize();
- bool verify_node_value(Node* n);
+ bool verify_Value_for(Node* n);
+ bool verify_Ideal_for(Node* n, bool can_reshape);
+ bool verify_Identity_for(Node* n);
+ void verify_empty_worklist(Node* n);
#endif
#ifndef PRODUCT
@@ -593,6 +596,14 @@ public:
// '-XX:VerifyIterativeGVN=10'
return ((VerifyIterativeGVN % 100) / 10) == 1;
}
+ static bool is_verify_Ideal() {
+ // '-XX:VerifyIterativeGVN=100'
+ return ((VerifyIterativeGVN % 1000) / 100) == 1;
+ }
+ static bool is_verify_Identity() {
+ // '-XX:VerifyIterativeGVN=1000'
+ return ((VerifyIterativeGVN % 10000) / 1000) == 1;
+ }
protected:
// Sub-quadratic implementation of '-XX:VerifyIterativeGVN=1' (Use-Def verification).
julong _verify_counter;
diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp
index 13acc0469eb..5ff2590a190 100644
--- a/src/hotspot/share/opto/vectorIntrinsics.cpp
+++ b/src/hotspot/share/opto/vectorIntrinsics.cpp
@@ -318,8 +318,10 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
- if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
- !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (opr == nullptr || !opr->is_con() ||
+ vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -587,7 +589,11 @@ bool LibraryCallKit::inline_vector_mask_operation() {
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
Node* mask = argument(4);
- if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
+ if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con() ||
+ oper == nullptr || !oper->is_con() ||
+ mask->is_top()) {
return false; // dead code
}
@@ -647,9 +653,11 @@ bool LibraryCallKit::inline_vector_frombits_coerced() {
// MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
const TypeInt* mode = gvn().type(argument(5))->isa_int();
- if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
- bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
- !vlen->is_con() || !mode->is_con()) {
+ if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con() ||
+ bits_type == nullptr ||
+ mode == nullptr || !mode->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -775,8 +783,10 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
- if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
- vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con() ||
+ from_ms == nullptr || !from_ms->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -983,9 +993,11 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
- if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
- vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
- elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
+ if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con() ||
+ from_ms == nullptr || !from_ms->is_con()) {
log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -1222,8 +1234,10 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
- if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
- vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con() ||
+ vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(2)->Opcode()],
@@ -1409,8 +1423,10 @@ bool LibraryCallKit::inline_vector_reduction() {
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
- if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
- !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (opr == nullptr || !opr->is_con() ||
+ vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -1547,8 +1563,10 @@ bool LibraryCallKit::inline_vector_test() {
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
- if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
- !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (cond == nullptr || !cond->is_con() ||
+ vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -2505,10 +2523,10 @@ bool LibraryCallKit::inline_vector_extract() {
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
const TypeInt* idx = gvn().type(argument(4))->isa_int();
- if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
- return false; // dead code
- }
- if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con() ||
+ idx == nullptr || !idx->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -2811,9 +2829,11 @@ bool LibraryCallKit::inline_vector_compress_expand() {
const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
- if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
- vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
- elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
+ if (opr == nullptr || !opr->is_con() ||
+ vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -2892,9 +2912,9 @@ bool LibraryCallKit::inline_index_vector() {
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
- if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
- vector_klass->const_oop() == nullptr || !vlen->is_con() ||
- elem_klass->const_oop() == nullptr) {
+ if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con() ) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -3026,8 +3046,9 @@ bool LibraryCallKit::inline_index_partially_in_upper_range() {
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
- if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
- mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
+ elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp
index 05ef64af704..2b40ca77198 100644
--- a/src/hotspot/share/opto/vectornode.cpp
+++ b/src/hotspot/share/opto/vectornode.cpp
@@ -1722,8 +1722,7 @@ Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_
if (cnt->Opcode() == Op_ConvI2L) {
cnt = cnt->in(1);
} else {
- assert(cnt->bottom_type()->isa_long() &&
- cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
+ assert(cnt->bottom_type()->isa_long(), "long type shift count expected");
cnt = phase->transform(new ConvL2INode(cnt));
}
}
diff --git a/src/hotspot/share/prims/jni.cpp b/src/hotspot/share/prims/jni.cpp
index 228244bbd05..5a43baeb8d8 100644
--- a/src/hotspot/share/prims/jni.cpp
+++ b/src/hotspot/share/prims/jni.cpp
@@ -2285,9 +2285,11 @@ JNI_ENTRY(jobjectArray, jni_NewObjectArray(JNIEnv *env, jsize length, jclass ele
jobjectArray ret = nullptr;
DT_RETURN_MARK(NewObjectArray, jobjectArray, (const jobjectArray&)ret);
Klass* ek = java_lang_Class::as_Klass(JNIHandles::resolve_non_null(elementClass));
- Klass* ak = ek->array_klass(CHECK_NULL);
- ObjArrayKlass::cast(ak)->initialize(CHECK_NULL);
- objArrayOop result = ObjArrayKlass::cast(ak)->allocate(length, CHECK_NULL);
+
+ // Make sure bottom_klass is initialized.
+ ek->initialize(CHECK_NULL);
+ objArrayOop result = oopFactory::new_objArray(ek, length, CHECK_NULL);
+
oop initial_value = JNIHandles::resolve(initialElement);
if (initial_value != nullptr) { // array already initialized with null
for (int index = 0; index < length; index++) {
diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp
index e50318c90b6..d669d7bf5ec 100644
--- a/src/hotspot/share/prims/jvm.cpp
+++ b/src/hotspot/share/prims/jvm.cpp
@@ -2961,6 +2961,15 @@ JVM_ENTRY(jobject, JVM_GetStackTrace(JNIEnv *env, jobject jthread))
return JNIHandles::make_local(THREAD, trace);
JVM_END
+JVM_ENTRY(jobject, JVM_CreateThreadSnapshot(JNIEnv* env, jobject jthread))
+#if INCLUDE_JVMTI
+ oop snapshot = ThreadSnapshotFactory::get_thread_snapshot(jthread, THREAD);
+ return JNIHandles::make_local(THREAD, snapshot);
+#else
+ return nullptr;
+#endif
+JVM_END
+
JVM_ENTRY(void, JVM_SetNativeThreadName(JNIEnv* env, jobject jthread, jstring name))
// We don't use a ThreadsListHandle here because the current thread
// must be alive.
diff --git a/src/hotspot/share/prims/jvmti.xml b/src/hotspot/share/prims/jvmti.xml
index a15dbb5703a..fef71737c78 100644
--- a/src/hotspot/share/prims/jvmti.xml
+++ b/src/hotspot/share/prims/jvmti.xml
@@ -12862,16 +12862,17 @@ myInit() {
parameters uniquely identify the current location
(where the exception was detected) and allow
the mapping to source file and line number when that information is
- available. The exception field identifies the thrown
+ available. The exception parameter identifies the thrown
exception object. The catch_method
and catch_location identify the location of the catch clause,
if any, that handles the thrown exception. If there is no such catch clause,
- each field is set to 0. There is no guarantee that the thread will ever
+ the catch_method is set to null and the catch_locationis set to 0.
+ There is no guarantee that the thread will ever
reach this catch clause. If there are native methods on the call stack
between the throw location and the catch clause, the exception may
be reset by one of those native methods.
- Similarly, exceptions that are reported as uncaught (catch_klass
- et al. set to 0) may in fact be caught by native code.
+ Similarly, exceptions that are reported as uncaught (catch_method
+ set to null) may in fact be caught by native code.
Agents can check for these occurrences by monitoring
events.
Note that finally clauses are implemented as catch and re-throw. Therefore they
@@ -12960,7 +12961,7 @@ myInit() {
available. For exceptions caught in a Java programming language method, the
exception object identifies the exception object. Exceptions
caught in native methods are not necessarily available by the time the
- exception catch is reported, so the exception field is set
+ exception catch is reported, so the exception parameter is set
to null.
jvmdi
diff --git a/src/hotspot/share/prims/jvmtiRedefineClasses.cpp b/src/hotspot/share/prims/jvmtiRedefineClasses.cpp
index af9d973d2f1..305af6df9be 100644
--- a/src/hotspot/share/prims/jvmtiRedefineClasses.cpp
+++ b/src/hotspot/share/prims/jvmtiRedefineClasses.cpp
@@ -3550,6 +3550,13 @@ void VM_RedefineClasses::set_new_constant_pool(
Array* new_fis = FieldInfoStream::create_FieldInfoStream(fields, java_fields, injected_fields, scratch_class->class_loader_data(), CHECK);
scratch_class->set_fieldinfo_stream(new_fis);
MetadataFactory::free_array(scratch_class->class_loader_data(), old_stream);
+
+ Array* old_table = scratch_class->fieldinfo_search_table();
+ Array* search_table = FieldInfoStream::create_search_table(scratch_class->constants(), new_fis, scratch_class->class_loader_data(), CHECK);
+ scratch_class->set_fieldinfo_search_table(search_table);
+ MetadataFactory::free_array(scratch_class->class_loader_data(), old_table);
+
+ DEBUG_ONLY(FieldInfoStream::validate_search_table(scratch_class->constants(), new_fis, search_table));
}
// Update constant pool indices in the inner classes info to use
diff --git a/src/hotspot/share/prims/jvmtiThreadState.hpp b/src/hotspot/share/prims/jvmtiThreadState.hpp
index cb0a779bd5a..62d0c45e337 100644
--- a/src/hotspot/share/prims/jvmtiThreadState.hpp
+++ b/src/hotspot/share/prims/jvmtiThreadState.hpp
@@ -77,7 +77,7 @@ class JvmtiEnvThreadStateIterator : public StackObj {
//
// Virtual Thread Mount State Transition (VTMS transition) mechanism
//
-class JvmtiVTMSTransitionDisabler {
+class JvmtiVTMSTransitionDisabler : public AnyObj {
private:
static volatile int _VTMS_transition_disable_for_one_count; // transitions for one virtual thread are disabled while it is positive
static volatile int _VTMS_transition_disable_for_all_count; // transitions for all virtual threads are disabled while it is positive
diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp
index c907ddb4885..002f737e788 100644
--- a/src/hotspot/share/prims/vectorSupport.cpp
+++ b/src/hotspot/share/prims/vectorSupport.cpp
@@ -28,6 +28,7 @@
#include "code/location.hpp"
#include "jni.h"
#include "jvm.h"
+#include "memory/oopFactory.hpp"
#include "oops/klass.inline.hpp"
#include "oops/typeArrayOop.inline.hpp"
#include "prims/vectorSupport.hpp"
@@ -109,9 +110,7 @@ Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* f
int elem_size = type2aelembytes(elem_bt);
// On-heap vector values are represented as primitive arrays.
- TypeArrayKlass* tak = Universe::typeArrayKlass(elem_bt);
-
- typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint
+ typeArrayOop arr = oopFactory::new_typeArray(elem_bt, num_elem, CHECK_NH); // safepoint
if (location.is_register()) {
// Value was in a callee-saved register.
diff --git a/src/hotspot/share/prims/whitebox.cpp b/src/hotspot/share/prims/whitebox.cpp
index f14c5efc65d..1507696c4d3 100644
--- a/src/hotspot/share/prims/whitebox.cpp
+++ b/src/hotspot/share/prims/whitebox.cpp
@@ -583,28 +583,6 @@ WB_ENTRY(jboolean, WB_G1HasRegionsToUncommit(JNIEnv* env, jobject o))
THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1HasRegionsToUncommit: G1 GC is not enabled");
WB_END
-#endif // INCLUDE_G1GC
-
-#if INCLUDE_PARALLELGC
-
-WB_ENTRY(jlong, WB_PSVirtualSpaceAlignment(JNIEnv* env, jobject o))
- if (UseParallelGC) {
- return GenAlignment;
- }
- THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_PSVirtualSpaceAlignment: Parallel GC is not enabled");
-WB_END
-
-WB_ENTRY(jlong, WB_PSHeapGenerationAlignment(JNIEnv* env, jobject o))
- if (UseParallelGC) {
- return GenAlignment;
- }
- THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_PSHeapGenerationAlignment: Parallel GC is not enabled");
-WB_END
-
-#endif // INCLUDE_PARALLELGC
-
-#if INCLUDE_G1GC
-
WB_ENTRY(jobject, WB_G1AuxiliaryMemoryUsage(JNIEnv* env))
if (UseG1GC) {
ResourceMark rm(THREAD);
@@ -794,7 +772,7 @@ class VM_WhiteBoxDeoptimizeFrames : public VM_WhiteBoxOperation {
if (_make_not_entrant) {
nmethod* nm = CodeCache::find_nmethod(f->pc());
assert(nm != nullptr, "did not find nmethod");
- nm->make_not_entrant("Whitebox deoptimization");
+ nm->make_not_entrant(nmethod::ChangeReason::whitebox_deoptimization);
}
++_result;
}
@@ -1097,6 +1075,22 @@ bool WhiteBox::validate_cgroup(bool cgroups_v2_enabled,
}
#endif
+bool WhiteBox::is_asan_enabled() {
+#ifdef ADDRESS_SANITIZER
+ return true;
+#else
+ return false;
+#endif
+}
+
+bool WhiteBox::is_ubsan_enabled() {
+#ifdef UNDEFINED_BEHAVIOR_SANITIZER
+ return true;
+#else
+ return false;
+#endif
+}
+
bool WhiteBox::compile_method(Method* method, int comp_level, int bci, JavaThread* THREAD) {
// Screen for unavailable/bad comp level or null method
AbstractCompiler* comp = CompileBroker::compiler(comp_level);
@@ -1908,6 +1902,14 @@ WB_ENTRY(jboolean, WB_IsMonitorInflated(JNIEnv* env, jobject wb, jobject obj))
return (jboolean) obj_oop->mark().has_monitor();
WB_END
+WB_ENTRY(jboolean, WB_IsAsanEnabled(JNIEnv* env))
+ return (jboolean) WhiteBox::is_asan_enabled();
+WB_END
+
+WB_ENTRY(jboolean, WB_IsUbsanEnabled(JNIEnv* env))
+ return (jboolean) WhiteBox::is_ubsan_enabled();
+WB_END
+
WB_ENTRY(jlong, WB_getInUseMonitorCount(JNIEnv* env, jobject wb))
return (jlong) WhiteBox::get_in_use_monitor_count();
WB_END
@@ -2773,10 +2775,6 @@ static JNINativeMethod methods[] = {
{CC"g1MemoryNodeIds", CC"()[I", (void*)&WB_G1MemoryNodeIds },
{CC"g1GetMixedGCInfo", CC"(I)[J", (void*)&WB_G1GetMixedGCInfo },
#endif // INCLUDE_G1GC
-#if INCLUDE_PARALLELGC
- {CC"psVirtualSpaceAlignment",CC"()J", (void*)&WB_PSVirtualSpaceAlignment},
- {CC"psHeapGenerationAlignment",CC"()J", (void*)&WB_PSHeapGenerationAlignment},
-#endif
{CC"NMTMalloc", CC"(J)J", (void*)&WB_NMTMalloc },
{CC"NMTMallocWithPseudoStack", CC"(JI)J", (void*)&WB_NMTMallocWithPseudoStack},
{CC"NMTMallocWithPseudoStackAndType", CC"(JII)J", (void*)&WB_NMTMallocWithPseudoStackAndType},
@@ -2908,6 +2906,8 @@ static JNINativeMethod methods[] = {
(void*)&WB_AddModuleExportsToAll },
{CC"deflateIdleMonitors", CC"()Z", (void*)&WB_DeflateIdleMonitors },
{CC"isMonitorInflated0", CC"(Ljava/lang/Object;)Z", (void*)&WB_IsMonitorInflated },
+ {CC"isAsanEnabled", CC"()Z", (void*)&WB_IsAsanEnabled },
+ {CC"isUbsanEnabled", CC"()Z", (void*)&WB_IsUbsanEnabled },
{CC"getInUseMonitorCount", CC"()J", (void*)&WB_getInUseMonitorCount },
{CC"getLockStackCapacity", CC"()I", (void*)&WB_getLockStackCapacity },
{CC"supportsRecursiveLightweightLocking", CC"()Z", (void*)&WB_supportsRecursiveLightweightLocking },
diff --git a/src/hotspot/share/prims/whitebox.hpp b/src/hotspot/share/prims/whitebox.hpp
index 4ba684fc09a..c20d35abbbb 100644
--- a/src/hotspot/share/prims/whitebox.hpp
+++ b/src/hotspot/share/prims/whitebox.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,9 @@ class WhiteBox : public AllStatic {
#ifdef LINUX
static bool validate_cgroup(bool cgroups_v2_enabled, const char* controllers_file, const char* proc_self_cgroup, const char* proc_self_mountinfo, u1* cg_flags);
#endif
+ // provide info about enabling of Address Sanitizer / Undefined Behavior Sanitizer
+ static bool is_asan_enabled();
+ static bool is_ubsan_enabled();
};
#endif // SHARE_PRIMS_WHITEBOX_HPP
diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp
index 7592d233f0c..a49116fd91d 100644
--- a/src/hotspot/share/runtime/arguments.cpp
+++ b/src/hotspot/share/runtime/arguments.cpp
@@ -528,10 +528,6 @@ static SpecialFlag const special_jvm_flags[] = {
{ "DynamicDumpSharedSpaces", JDK_Version::jdk(18), JDK_Version::jdk(19), JDK_Version::undefined() },
{ "RequireSharedSpaces", JDK_Version::jdk(18), JDK_Version::jdk(19), JDK_Version::undefined() },
{ "UseSharedSpaces", JDK_Version::jdk(18), JDK_Version::jdk(19), JDK_Version::undefined() },
-#ifdef LINUX
- { "UseLinuxPosixThreadCPUClocks", JDK_Version::jdk(24), JDK_Version::jdk(25), JDK_Version::jdk(26) },
- { "UseOprofile", JDK_Version::jdk(25), JDK_Version::jdk(26), JDK_Version::jdk(27) },
-#endif
{ "LockingMode", JDK_Version::jdk(24), JDK_Version::jdk(26), JDK_Version::jdk(27) },
#ifdef _LP64
{ "UseCompressedClassPointers", JDK_Version::jdk(25), JDK_Version::jdk(26), JDK_Version::undefined() },
@@ -541,7 +537,9 @@ static SpecialFlag const special_jvm_flags[] = {
// -------------- Obsolete Flags - sorted by expired_in --------------
- { "PerfDataSamplingInterval", JDK_Version::undefined(), JDK_Version::jdk(25), JDK_Version::jdk(26) },
+#ifdef LINUX
+ { "UseOprofile", JDK_Version::jdk(25), JDK_Version::jdk(26), JDK_Version::jdk(27) },
+#endif
{ "MetaspaceReclaimPolicy", JDK_Version::undefined(), JDK_Version::jdk(21), JDK_Version::undefined() },
{ "ZGenerational", JDK_Version::jdk(23), JDK_Version::jdk(24), JDK_Version::undefined() },
{ "ZMarkStackSpaceLimit", JDK_Version::undefined(), JDK_Version::jdk(25), JDK_Version::undefined() },
diff --git a/src/hotspot/share/runtime/deoptimization.cpp b/src/hotspot/share/runtime/deoptimization.cpp
index 4042bb01c58..ed84f2b294f 100644
--- a/src/hotspot/share/runtime/deoptimization.cpp
+++ b/src/hotspot/share/runtime/deoptimization.cpp
@@ -1274,11 +1274,11 @@ bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap*
assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
int len = sv->field_size() / type2size[ak->element_type()];
InternalOOMEMark iom(THREAD);
- obj = ak->allocate(len, THREAD);
+ obj = ak->allocate_instance(len, THREAD);
} else if (k->is_objArray_klass()) {
ObjArrayKlass* ak = ObjArrayKlass::cast(k);
InternalOOMEMark iom(THREAD);
- obj = ak->allocate(sv->field_size(), THREAD);
+ obj = ak->allocate_instance(sv->field_size(), THREAD);
}
if (obj == nullptr) {
@@ -1826,7 +1826,7 @@ void Deoptimization::deoptimize(JavaThread* thread, frame fr, DeoptReason reason
#if INCLUDE_JVMCI
address Deoptimization::deoptimize_for_missing_exception_handler(nmethod* nm) {
// there is no exception handler for this pc => deoptimize
- nm->make_not_entrant("missing exception handler");
+ nm->make_not_entrant(nmethod::ChangeReason::missing_exception_handler);
// Use Deoptimization::deoptimize for all of its side-effects:
// gathering traps statistics, logging...
@@ -2455,7 +2455,7 @@ JRT_ENTRY(void, Deoptimization::uncommon_trap_inner(JavaThread* current, jint tr
// Recompile
if (make_not_entrant) {
- if (!nm->make_not_entrant("uncommon trap")) {
+ if (!nm->make_not_entrant(nmethod::ChangeReason::uncommon_trap)) {
return; // the call did not change nmethod's state
}
diff --git a/src/hotspot/share/runtime/flags/jvmFlagConstraintsCompiler.cpp b/src/hotspot/share/runtime/flags/jvmFlagConstraintsCompiler.cpp
index d9972b21ea1..18aa4a56d71 100644
--- a/src/hotspot/share/runtime/flags/jvmFlagConstraintsCompiler.cpp
+++ b/src/hotspot/share/runtime/flags/jvmFlagConstraintsCompiler.cpp
@@ -299,8 +299,9 @@ JVMFlag::Error TypeProfileLevelConstraintFunc(uint value, bool verbose) {
}
JVMFlag::Error VerifyIterativeGVNConstraintFunc(uint value, bool verbose) {
+ const int max_modes = 4;
uint original_value = value;
- for (int i = 0; i < 2; i++) {
+ for (int i = 0; i < max_modes; i++) {
if (value % 10 > 1) {
JVMFlag::printError(verbose,
"Invalid value (" UINT32_FORMAT ") "
@@ -312,7 +313,7 @@ JVMFlag::Error VerifyIterativeGVNConstraintFunc(uint value, bool verbose) {
if (value != 0) {
JVMFlag::printError(verbose,
"Invalid value (" UINT32_FORMAT ") "
- "for VerifyIterativeGVN: maximal 2 digits\n", original_value);
+ "for VerifyIterativeGVN: maximal %d digits\n", original_value, max_modes);
return JVMFlag::VIOLATES_CONSTRAINT;
}
return JVMFlag::SUCCESS;
diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp
index 290ce0d3cba..75736d0dc7d 100644
--- a/src/hotspot/share/runtime/globals.hpp
+++ b/src/hotspot/share/runtime/globals.hpp
@@ -2005,6 +2005,10 @@ const int ObjectAlignmentInBytes = 8;
product(bool, UseThreadsLockThrottleLock, true, DIAGNOSTIC, \
"Use an extra lock during Thread start and exit to alleviate" \
"contention on Threads_lock.") \
+ \
+ develop(uint, BinarySearchThreshold, 16, \
+ "Minimal number of elements in a sorted collection to prefer" \
+ "binary search over simple linear search." ) \
// end of RUNTIME_FLAGS
diff --git a/src/hotspot/share/runtime/handshake.cpp b/src/hotspot/share/runtime/handshake.cpp
index 6bed5e9d546..2c827b61602 100644
--- a/src/hotspot/share/runtime/handshake.cpp
+++ b/src/hotspot/share/runtime/handshake.cpp
@@ -465,9 +465,7 @@ HandshakeState::HandshakeState(JavaThread* target) :
_queue(),
_lock(Monitor::nosafepoint, "HandshakeState_lock"),
_active_handshaker(),
- _async_exceptions_blocked(false),
- _suspended(false),
- _async_suspend_handshake(false) {
+ _async_exceptions_blocked(false) {
}
HandshakeState::~HandshakeState() {
@@ -699,128 +697,8 @@ HandshakeState::ProcessResult HandshakeState::try_process(HandshakeOperation* ma
return op == match_op ? HandshakeState::_succeeded : HandshakeState::_processed;
}
-void HandshakeState::do_self_suspend() {
- assert(Thread::current() == _handshakee, "should call from _handshakee");
- assert(_lock.owned_by_self(), "Lock must be held");
- assert(!_handshakee->has_last_Java_frame() || _handshakee->frame_anchor()->walkable(), "should have walkable stack");
- assert(_handshakee->thread_state() == _thread_blocked, "Caller should have transitioned to _thread_blocked");
-
- while (is_suspended()) {
- log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " suspended", p2i(_handshakee));
- _lock.wait_without_safepoint_check();
- }
- log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " resumed", p2i(_handshakee));
-}
-
-// This is the closure that prevents a suspended JavaThread from
-// escaping the suspend request.
-class ThreadSelfSuspensionHandshake : public AsyncHandshakeClosure {
- public:
- ThreadSelfSuspensionHandshake() : AsyncHandshakeClosure("ThreadSelfSuspensionHandshake") {}
- void do_thread(Thread* thr) {
- JavaThread* current = JavaThread::cast(thr);
- assert(current == Thread::current(), "Must be self executed.");
- JavaThreadState jts = current->thread_state();
-
- current->set_thread_state(_thread_blocked);
- current->handshake_state()->do_self_suspend();
- current->set_thread_state(jts);
- current->handshake_state()->set_async_suspend_handshake(false);
- }
- virtual bool is_suspend() { return true; }
-};
-
-bool HandshakeState::suspend_with_handshake(bool register_vthread_SR) {
- assert(_handshakee->threadObj() != nullptr, "cannot suspend with a null threadObj");
- if (_handshakee->is_exiting()) {
- log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " exiting", p2i(_handshakee));
- return false;
- }
- if (has_async_suspend_handshake()) {
- if (is_suspended()) {
- // Target is already suspended.
- log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " already suspended", p2i(_handshakee));
- return false;
- } else {
- // Target is going to wake up and leave suspension.
- // Let's just stop the thread from doing that.
- log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " re-suspended", p2i(_handshakee));
- set_suspended(true, register_vthread_SR);
- return true;
- }
- }
- // no suspend request
- assert(!is_suspended(), "cannot be suspended without a suspend request");
- // Thread is safe, so it must execute the request, thus we can count it as suspended
- // from this point.
- set_suspended(true, register_vthread_SR);
- set_async_suspend_handshake(true);
- log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " suspended, arming ThreadSuspension", p2i(_handshakee));
- ThreadSelfSuspensionHandshake* ts = new ThreadSelfSuspensionHandshake();
- Handshake::execute(ts, _handshakee);
- return true;
-}
-
-// This is the closure that synchronously honors the suspend request.
-class SuspendThreadHandshake : public HandshakeClosure {
- bool _register_vthread_SR;
- bool _did_suspend;
-public:
- SuspendThreadHandshake(bool register_vthread_SR) : HandshakeClosure("SuspendThread"),
- _register_vthread_SR(register_vthread_SR), _did_suspend(false) {}
- void do_thread(Thread* thr) {
- JavaThread* target = JavaThread::cast(thr);
- _did_suspend = target->handshake_state()->suspend_with_handshake(_register_vthread_SR);
- }
- bool did_suspend() { return _did_suspend; }
-};
-
-bool HandshakeState::suspend(bool register_vthread_SR) {
- JVMTI_ONLY(assert(!_handshakee->is_in_VTMS_transition(), "no suspend allowed in VTMS transition");)
- JavaThread* self = JavaThread::current();
- if (_handshakee == self) {
- // If target is the current thread we can bypass the handshake machinery
- // and just suspend directly
- ThreadBlockInVM tbivm(self);
- MutexLocker ml(&_lock, Mutex::_no_safepoint_check_flag);
- set_suspended(true, register_vthread_SR);
- do_self_suspend();
- return true;
- } else {
- SuspendThreadHandshake st(register_vthread_SR);
- Handshake::execute(&st, _handshakee);
- return st.did_suspend();
- }
-}
-
-bool HandshakeState::resume(bool register_vthread_SR) {
- MutexLocker ml(&_lock, Mutex::_no_safepoint_check_flag);
- if (!is_suspended()) {
- assert(!_handshakee->is_suspended(), "cannot be suspended without a suspend request");
- return false;
- }
- // Resume the thread.
- set_suspended(false, register_vthread_SR);
- _lock.notify();
- return true;
-}
-
-void HandshakeState::set_suspended(bool is_suspend, bool register_vthread_SR) {
-#if INCLUDE_JVMTI
- if (register_vthread_SR) {
- assert(_handshakee->is_vthread_mounted(), "sanity check");
- if (is_suspend) {
- JvmtiVTSuspender::register_vthread_suspend(_handshakee->vthread());
- } else {
- JvmtiVTSuspender::register_vthread_resume(_handshakee->vthread());
- }
- }
-#endif
- Atomic::store(&_suspended, is_suspend);
-}
-
void HandshakeState::handle_unsafe_access_error() {
- if (is_suspended()) {
+ if (_handshakee->is_suspended()) {
// A suspend handshake was added to the queue after the
// unsafe access error. Since the suspender has already
// considered this JT as suspended and assumes it won't go
diff --git a/src/hotspot/share/runtime/handshake.hpp b/src/hotspot/share/runtime/handshake.hpp
index 9e963003053..ca9cef76f5d 100644
--- a/src/hotspot/share/runtime/handshake.hpp
+++ b/src/hotspot/share/runtime/handshake.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,8 +35,6 @@
class HandshakeOperation;
class AsyncHandshakeOperation;
class JavaThread;
-class SuspendThreadHandshake;
-class ThreadSelfSuspensionHandshake;
class UnsafeAccessErrorHandshake;
class ThreadsListHandle;
@@ -88,8 +86,6 @@ class JvmtiRawMonitor;
// operation is only done by either VMThread/Handshaker on behalf of the
// JavaThread or by the target JavaThread itself.
class HandshakeState {
- friend ThreadSelfSuspensionHandshake;
- friend SuspendThreadHandshake;
friend UnsafeAccessErrorHandshake;
friend JavaThread;
// This a back reference to the JavaThread,
@@ -98,7 +94,7 @@ class HandshakeState {
// The queue containing handshake operations to be performed on _handshakee.
FilterQueue _queue;
// Provides mutual exclusion to this state and queue. Also used for
- // JavaThread suspend/resume operations.
+ // JavaThread suspend/resume operations performed by SuspendResumeManager.
Monitor _lock;
// Set to the thread executing the handshake operation.
Thread* volatile _active_handshaker;
@@ -160,31 +156,5 @@ class HandshakeState {
bool async_exceptions_blocked() { return _async_exceptions_blocked; }
void set_async_exceptions_blocked(bool b) { _async_exceptions_blocked = b; }
void handle_unsafe_access_error();
-
- // Suspend/resume support
- private:
- // This flag is true when the thread owning this
- // HandshakeState (the _handshakee) is suspended.
- volatile bool _suspended;
- // This flag is true while there is async handshake (trap)
- // on queue. Since we do only need one, we can reuse it if
- // thread gets suspended again (after a resume)
- // and we have not yet processed it.
- bool _async_suspend_handshake;
-
- // Called from the suspend handshake.
- bool suspend_with_handshake(bool register_vthread_SR);
- // Called from the async handshake (the trap)
- // to stop a thread from continuing execution when suspended.
- void do_self_suspend();
-
- bool is_suspended() { return Atomic::load(&_suspended); }
- void set_suspended(bool to, bool register_vthread_SR);
- bool has_async_suspend_handshake() { return _async_suspend_handshake; }
- void set_async_suspend_handshake(bool to) { _async_suspend_handshake = to; }
-
- bool suspend(bool register_vthread_SR);
- bool resume(bool register_vthread_SR);
};
-
#endif // SHARE_RUNTIME_HANDSHAKE_HPP
diff --git a/src/hotspot/share/runtime/javaThread.cpp b/src/hotspot/share/runtime/javaThread.cpp
index 57f93f87d47..37ecb1c6f9a 100644
--- a/src/hotspot/share/runtime/javaThread.cpp
+++ b/src/hotspot/share/runtime/javaThread.cpp
@@ -498,6 +498,7 @@ JavaThread::JavaThread(MemTag mem_tag) :
_pending_interrupted_exception(false),
_handshake(this),
+ _suspend_resume_manager(this, &_handshake._lock),
_popframe_preserved_args(nullptr),
_popframe_preserved_args_size(0),
@@ -1200,13 +1201,13 @@ bool JavaThread::java_suspend(bool register_vthread_SR) {
guarantee(Thread::is_JavaThread_protected(/* target */ this),
"target JavaThread is not protected in calling context.");
- return this->handshake_state()->suspend(register_vthread_SR);
+ return this->suspend_resume_manager()->suspend(register_vthread_SR);
}
bool JavaThread::java_resume(bool register_vthread_SR) {
guarantee(Thread::is_JavaThread_protected_by_TLH(/* target */ this),
"missing ThreadsListHandle in calling context.");
- return this->handshake_state()->resume(register_vthread_SR);
+ return this->suspend_resume_manager()->resume(register_vthread_SR);
}
// Wait for another thread to perform object reallocation and relocking on behalf of
@@ -1337,7 +1338,7 @@ void JavaThread::make_zombies() {
// it is a Java nmethod
nmethod* nm = CodeCache::find_nmethod(fst.current()->pc());
assert(nm != nullptr, "did not find nmethod");
- nm->make_not_entrant("zombie");
+ nm->make_not_entrant(nmethod::ChangeReason::zombie);
}
}
}
diff --git a/src/hotspot/share/runtime/javaThread.hpp b/src/hotspot/share/runtime/javaThread.hpp
index 968dfd0ce48..af46492622d 100644
--- a/src/hotspot/share/runtime/javaThread.hpp
+++ b/src/hotspot/share/runtime/javaThread.hpp
@@ -40,6 +40,7 @@
#include "runtime/safepointMechanism.hpp"
#include "runtime/stackWatermarkSet.hpp"
#include "runtime/stackOverflow.hpp"
+#include "runtime/suspendResumeManager.hpp"
#include "runtime/thread.hpp"
#include "runtime/threadHeapSampler.hpp"
#include "runtime/threadIdentifier.hpp"
@@ -694,9 +695,13 @@ private:
// Suspend/resume support for JavaThread
// higher-level suspension/resume logic called by the public APIs
+private:
+ SuspendResumeManager _suspend_resume_manager;
+public:
bool java_suspend(bool register_vthread_SR);
bool java_resume(bool register_vthread_SR);
- bool is_suspended() { return _handshake.is_suspended(); }
+ bool is_suspended() { return _suspend_resume_manager.is_suspended(); }
+ SuspendResumeManager* suspend_resume_manager() { return &_suspend_resume_manager; }
// Check for async exception in addition to safepoint.
static void check_special_condition_for_native_trans(JavaThread *thread);
diff --git a/src/hotspot/share/runtime/reflection.cpp b/src/hotspot/share/runtime/reflection.cpp
index 9e165f76829..2a0af9d2d09 100644
--- a/src/hotspot/share/runtime/reflection.cpp
+++ b/src/hotspot/share/runtime/reflection.cpp
@@ -320,9 +320,16 @@ void Reflection::array_set(jvalue* value, arrayOop a, int index, BasicType value
}
}
+
+// Conversion
+static BasicType basic_type_mirror_to_basic_type(oop basic_type_mirror) {
+ assert(java_lang_Class::is_primitive(basic_type_mirror),
+ "just checking");
+ return java_lang_Class::primitive_type(basic_type_mirror);
+}
+
static Klass* basic_type_mirror_to_arrayklass(oop basic_type_mirror, TRAPS) {
- assert(java_lang_Class::is_primitive(basic_type_mirror), "just checking");
- BasicType type = java_lang_Class::primitive_type(basic_type_mirror);
+ BasicType type = basic_type_mirror_to_basic_type(basic_type_mirror);
if (type == T_VOID) {
THROW_NULL(vmSymbols::java_lang_IllegalArgumentException());
}
@@ -339,8 +346,11 @@ arrayOop Reflection::reflect_new_array(oop element_mirror, jint length, TRAPS) {
THROW_MSG_NULL(vmSymbols::java_lang_NegativeArraySizeException(), err_msg("%d", length));
}
if (java_lang_Class::is_primitive(element_mirror)) {
- Klass* tak = basic_type_mirror_to_arrayklass(element_mirror, CHECK_NULL);
- return TypeArrayKlass::cast(tak)->allocate(length, THREAD);
+ BasicType type = basic_type_mirror_to_basic_type(element_mirror);
+ if (type == T_VOID) {
+ THROW_NULL(vmSymbols::java_lang_IllegalArgumentException());
+ }
+ return oopFactory::new_typeArray(type, length, CHECK_NULL);
} else {
Klass* k = java_lang_Class::as_Klass(element_mirror);
if (k->is_array_klass() && ArrayKlass::cast(k)->dimension() >= MAX_DIM) {
@@ -907,13 +917,6 @@ static methodHandle resolve_interface_call(InstanceKlass* klass,
return methodHandle(THREAD, info.selected_method());
}
-// Conversion
-static BasicType basic_type_mirror_to_basic_type(oop basic_type_mirror) {
- assert(java_lang_Class::is_primitive(basic_type_mirror),
- "just checking");
- return java_lang_Class::primitive_type(basic_type_mirror);
-}
-
// Narrowing of basic types. Used to create correct jvalues for
// boolean, byte, char and short return return values from interpreter
// which are returned as ints. Throws IllegalArgumentException.
diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp
index edb33b3500a..9c710fc98e4 100644
--- a/src/hotspot/share/runtime/sharedRuntime.cpp
+++ b/src/hotspot/share/runtime/sharedRuntime.cpp
@@ -2199,10 +2199,11 @@ class AdapterFingerPrint : public MetaspaceObj {
}
// Private construtor. Use allocate() to get an instance.
- AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) {
+ AdapterFingerPrint(int total_args_passed, BasicType* sig_bt, int len) {
int* data = data_pointer();
// Pack the BasicTypes with 8 per int
- _length = length(total_args_passed);
+ assert(len == length(total_args_passed), "sanity");
+ _length = len;
int sig_index = 0;
for (int index = 0; index < _length; index++) {
int value = 0;
@@ -2217,16 +2218,15 @@ class AdapterFingerPrint : public MetaspaceObj {
// Call deallocate instead
~AdapterFingerPrint() {
- FreeHeap(this);
+ ShouldNotCallThis();
}
static int length(int total_args) {
return (total_args + (_basic_types_per_int-1)) / _basic_types_per_int;
}
- static int compute_size(int total_args_passed, BasicType* sig_bt) {
- int len = length(total_args_passed);
- return sizeof(AdapterFingerPrint) + (len * sizeof(int));
+ static int compute_size_in_words(int len) {
+ return (int)heap_word_size(sizeof(AdapterFingerPrint) + (len * sizeof(int)));
}
// Remap BasicTypes that are handled equivalently by the adapters.
@@ -2289,12 +2289,15 @@ class AdapterFingerPrint : public MetaspaceObj {
public:
static AdapterFingerPrint* allocate(int total_args_passed, BasicType* sig_bt) {
- int size_in_bytes = compute_size(total_args_passed, sig_bt);
- return new (size_in_bytes) AdapterFingerPrint(total_args_passed, sig_bt);
+ int len = length(total_args_passed);
+ int size_in_bytes = BytesPerWord * compute_size_in_words(len);
+ AdapterFingerPrint* afp = new (size_in_bytes) AdapterFingerPrint(total_args_passed, sig_bt, len);
+ assert((afp->size() * BytesPerWord) == size_in_bytes, "should match");
+ return afp;
}
static void deallocate(AdapterFingerPrint* fp) {
- fp->~AdapterFingerPrint();
+ FreeHeap(fp);
}
int value(int index) {
@@ -2418,7 +2421,7 @@ class AdapterFingerPrint : public MetaspaceObj {
// methods required by virtue of being a MetaspaceObj
void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ }
- int size() const { return (int)heap_word_size(sizeof(AdapterFingerPrint) + (_length * sizeof(int))); }
+ int size() const { return compute_size_in_words(_length); }
MetaspaceObj::Type type() const { return AdapterFingerPrintType; }
static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) {
diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp
index 86074ac7aca..eb2ffb82e5a 100644
--- a/src/hotspot/share/runtime/sharedRuntime.hpp
+++ b/src/hotspot/share/runtime/sharedRuntime.hpp
@@ -711,6 +711,7 @@ class AdapterHandlerEntry : public MetaspaceObj {
// Dummy argument is used to avoid C++ warning about using
// deleted opearator MetaspaceObj::delete().
void* operator new(size_t size, size_t dummy) throw() {
+ assert(size == BytesPerWord * heap_word_size(sizeof(AdapterHandlerEntry)), "should match");
void* p = AllocateHeap(size, mtCode);
memset(p, 0, size);
return p;
diff --git a/src/hotspot/share/runtime/suspendResumeManager.cpp b/src/hotspot/share/runtime/suspendResumeManager.cpp
new file mode 100644
index 00000000000..fd14f73f553
--- /dev/null
+++ b/src/hotspot/share/runtime/suspendResumeManager.cpp
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "logging/log.hpp"
+#include "logging/logStream.hpp"
+#include "memory/resourceArea.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/handshake.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+#include "runtime/javaThread.inline.hpp"
+#include "runtime/suspendResumeManager.hpp"
+
+// This is the closure that prevents a suspended JavaThread from
+// escaping the suspend request.
+class ThreadSelfSuspensionHandshake : public AsyncHandshakeClosure {
+public:
+ ThreadSelfSuspensionHandshake() : AsyncHandshakeClosure("ThreadSelfSuspensionHandshake") {}
+ void do_thread(Thread* thr) {
+ JavaThread* current = JavaThread::cast(thr);
+ assert(current == Thread::current(), "Must be self executed.");
+ JavaThreadState jts = current->thread_state();
+
+ current->set_thread_state(_thread_blocked);
+ current->suspend_resume_manager()->do_owner_suspend();
+ current->set_thread_state(jts);
+ current->suspend_resume_manager()->set_async_suspend_handshake(false);
+ }
+ virtual bool is_suspend() { return true; }
+};
+
+// This is the closure that synchronously honors the suspend request.
+class SuspendThreadHandshake : public HandshakeClosure {
+ bool _register_vthread_SR;
+ bool _did_suspend;
+public:
+ SuspendThreadHandshake(bool register_vthread_SR) : HandshakeClosure("SuspendThread"),
+ _register_vthread_SR(register_vthread_SR), _did_suspend(false) {
+ }
+ void do_thread(Thread* thr) {
+ JavaThread* target = JavaThread::cast(thr);
+ _did_suspend = target->suspend_resume_manager()->suspend_with_handshake(_register_vthread_SR);
+ }
+ bool did_suspend() { return _did_suspend; }
+};
+
+void SuspendResumeManager::set_suspended(bool is_suspend, bool register_vthread_SR) {
+#if INCLUDE_JVMTI
+ if (register_vthread_SR) {
+ assert(_target->is_vthread_mounted(), "sanity check");
+ if (is_suspend) {
+ JvmtiVTSuspender::register_vthread_suspend(_target->vthread());
+ }
+ else {
+ JvmtiVTSuspender::register_vthread_resume(_target->vthread());
+ }
+ }
+#endif
+ Atomic::store(&_suspended, is_suspend);
+}
+
+bool SuspendResumeManager::suspend(bool register_vthread_SR) {
+ JVMTI_ONLY(assert(!_target->is_in_VTMS_transition(), "no suspend allowed in VTMS transition");)
+ JavaThread* self = JavaThread::current();
+ if (_target == self) {
+ // If target is the current thread we can bypass the handshake machinery
+ // and just suspend directly
+ ThreadBlockInVM tbivm(self);
+ MutexLocker ml(_state_lock, Mutex::_no_safepoint_check_flag);
+ set_suspended(true, register_vthread_SR);
+ do_owner_suspend();
+ return true;
+ } else {
+ SuspendThreadHandshake st(register_vthread_SR);
+ Handshake::execute(&st, _target);
+ return st.did_suspend();
+ }
+}
+
+bool SuspendResumeManager::resume(bool register_vthread_SR) {
+ MutexLocker ml(_state_lock, Mutex::_no_safepoint_check_flag);
+ if (!is_suspended()) {
+ assert(!_target->is_suspended(), "cannot be suspended without a suspend request");
+ return false;
+ }
+ // Resume the thread.
+ set_suspended(false, register_vthread_SR);
+ _state_lock->notify();
+ return true;
+}
+
+void SuspendResumeManager::do_owner_suspend() {
+ assert(Thread::current() == _target, "should call from _target");
+ assert(_state_lock->owned_by_self(), "Lock must be held");
+ assert(!_target->has_last_Java_frame() || _target->frame_anchor()->walkable(), "should have walkable stack");
+ assert(_target->thread_state() == _thread_blocked, "Caller should have transitioned to _thread_blocked");
+
+ while (is_suspended()) {
+ log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " suspended", p2i(_target));
+ _state_lock->wait_without_safepoint_check();
+ }
+ log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " resumed", p2i(_target));
+}
+
+bool SuspendResumeManager::suspend_with_handshake(bool register_vthread_SR) {
+ assert(_target->threadObj() != nullptr, "cannot suspend with a null threadObj");
+ if (_target->is_exiting()) {
+ log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " exiting", p2i(_target));
+ return false;
+ }
+ if (has_async_suspend_handshake()) {
+ if (is_suspended()) {
+ // Target is already suspended.
+ log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " already suspended", p2i(_target));
+ return false;
+ } else {
+ // Target is going to wake up and leave suspension.
+ // Let's just stop the thread from doing that.
+ log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " re-suspended", p2i(_target));
+ set_suspended(true, register_vthread_SR);
+ return true;
+ }
+ }
+ // no suspend request
+ assert(!is_suspended(), "cannot be suspended without a suspend request");
+ // Thread is safe, so it must execute the request, thus we can count it as suspended
+ // from this point.
+ set_suspended(true, register_vthread_SR);
+ set_async_suspend_handshake(true);
+ log_trace(thread, suspend)("JavaThread:" INTPTR_FORMAT " suspended, arming ThreadSuspension", p2i(_target));
+ ThreadSelfSuspensionHandshake* ts = new ThreadSelfSuspensionHandshake();
+ Handshake::execute(ts, _target);
+ return true;
+}
+
+SuspendResumeManager::SuspendResumeManager(JavaThread* thread, Monitor* state_lock) : _target(thread), _state_lock(state_lock), _suspended(false), _async_suspend_handshake(false) {}
diff --git a/src/hotspot/share/runtime/suspendResumeManager.hpp b/src/hotspot/share/runtime/suspendResumeManager.hpp
new file mode 100644
index 00000000000..fed3b34055e
--- /dev/null
+++ b/src/hotspot/share/runtime/suspendResumeManager.hpp
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_RUNTIME_SUSPENDRESUMEMANAGER_HPP
+#define SHARE_RUNTIME_SUSPENDRESUMEMANAGER_HPP
+
+class SuspendThreadHandshake;
+class ThreadSelfSuspensionHandshake;
+
+class SuspendResumeManager {
+ friend SuspendThreadHandshake;
+ friend ThreadSelfSuspensionHandshake;
+ friend JavaThread;
+
+ JavaThread* _target;
+ Monitor* _state_lock;
+
+ SuspendResumeManager(JavaThread* thread, Monitor* state_lock);
+
+ // This flag is true when the thread owning this
+ // SuspendResumeManager (the _target) is suspended.
+ volatile bool _suspended;
+ // This flag is true while there is async handshake (trap)
+ // on queue. Since we do only need one, we can reuse it if
+ // thread gets suspended again (after a resume)
+ // and we have not yet processed it.
+ bool _async_suspend_handshake;
+
+ bool suspend(bool register_vthread_SR);
+ bool resume(bool register_vthread_SR);
+
+ // Called from the async handshake (the trap)
+ // to stop a thread from continuing execution when suspended.
+ void do_owner_suspend();
+
+ // Called from the suspend handshake.
+ bool suspend_with_handshake(bool register_vthread_SR);
+
+ void set_suspended(bool to, bool register_vthread_SR);
+
+ bool is_suspended() {
+ return Atomic::load(&_suspended);
+ }
+
+ bool has_async_suspend_handshake() { return _async_suspend_handshake; }
+ void set_async_suspend_handshake(bool to) { _async_suspend_handshake = to; }
+};
+
+#endif // SHARE_RUNTIME_SUSPENDRESUMEMANAGER_HPP
diff --git a/src/hotspot/share/runtime/thread.hpp b/src/hotspot/share/runtime/thread.hpp
index f3a06d5efd2..772ef7bbe82 100644
--- a/src/hotspot/share/runtime/thread.hpp
+++ b/src/hotspot/share/runtime/thread.hpp
@@ -78,6 +78,7 @@ class JavaThread;
// - WorkerThread
// - WatcherThread
// - JfrThreadSampler
+// - JfrCPUSamplerThread
// - LogAsyncWriter
//
// All Thread subclasses must be either JavaThread or NonJavaThread.
diff --git a/src/hotspot/share/runtime/vmOperation.hpp b/src/hotspot/share/runtime/vmOperation.hpp
index 50d85944485..ac5d37381f9 100644
--- a/src/hotspot/share/runtime/vmOperation.hpp
+++ b/src/hotspot/share/runtime/vmOperation.hpp
@@ -115,6 +115,8 @@
template(JFROldObject) \
template(JvmtiPostObjectFree) \
template(RendezvousGCThreads) \
+ template(JFRInitializeCPUTimeSampler) \
+ template(JFRTerminateCPUTimeSampler) \
template(ReinitializeMDO)
class Thread;
diff --git a/src/hotspot/share/services/attachListener.cpp b/src/hotspot/share/services/attachListener.cpp
index d13a2e70b2b..92d3c302ded 100644
--- a/src/hotspot/share/services/attachListener.cpp
+++ b/src/hotspot/share/services/attachListener.cpp
@@ -172,8 +172,8 @@ volatile AttachListenerState AttachListener::_state = AL_NOT_INITIALIZED;
AttachAPIVersion AttachListener::_supported_version = ATTACH_API_V1;
-// Default is false (if jdk.attach.vm.streaming property is not set).
-bool AttachListener::_default_streaming_output = false;
+// Default is true (if jdk.attach.vm.streaming property is not set).
+bool AttachListener::_default_streaming_output = true;
static bool get_bool_sys_prop(const char* name, bool default_value, TRAPS) {
ResourceMark rm(THREAD);
diff --git a/src/hotspot/share/services/diagnosticCommand.cpp b/src/hotspot/share/services/diagnosticCommand.cpp
index 26a762fa109..8b6fa392c95 100644
--- a/src/hotspot/share/services/diagnosticCommand.cpp
+++ b/src/hotspot/share/services/diagnosticCommand.cpp
@@ -127,7 +127,9 @@ void DCmd::register_dcmds(){
DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false));
#endif // INCLUDE_JVMTI
DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false));
+#if INCLUDE_JVMTI
DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false));
+#endif // INCLUDE_JVMTI
DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false));
DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false));
DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false));
diff --git a/src/hotspot/share/services/threadService.cpp b/src/hotspot/share/services/threadService.cpp
index d320e17fafb..8e0c955bff8 100644
--- a/src/hotspot/share/services/threadService.cpp
+++ b/src/hotspot/share/services/threadService.cpp
@@ -34,21 +34,25 @@
#include "nmt/memTag.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klass.inline.hpp"
+#include "oops/method.inline.hpp"
#include "oops/objArrayKlass.hpp"
#include "oops/objArrayOop.inline.hpp"
#include "oops/oop.inline.hpp"
#include "oops/oopHandle.inline.hpp"
#include "prims/jvmtiRawMonitor.hpp"
+#include "prims/jvmtiThreadState.hpp"
#include "runtime/atomic.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/init.hpp"
+#include "runtime/javaCalls.hpp"
#include "runtime/javaThread.inline.hpp"
+#include "runtime/jniHandles.inline.hpp"
#include "runtime/objectMonitor.inline.hpp"
-#include "runtime/synchronizer.hpp"
+#include "runtime/synchronizer.inline.hpp"
#include "runtime/thread.inline.hpp"
#include "runtime/threads.hpp"
#include "runtime/threadSMR.inline.hpp"
-#include "runtime/vframe.hpp"
+#include "runtime/vframe.inline.hpp"
#include "runtime/vmThread.hpp"
#include "runtime/vmOperations.hpp"
#include "services/threadService.hpp"
@@ -1115,3 +1119,431 @@ ThreadsListEnumerator::ThreadsListEnumerator(Thread* cur_thread,
_threads_array->append(h);
}
}
+
+
+// jdk.internal.vm.ThreadSnapshot support
+#if INCLUDE_JVMTI
+
+class GetThreadSnapshotClosure: public HandshakeClosure {
+private:
+ static OopStorage* oop_storage() {
+ assert(_thread_service_storage != nullptr, "sanity");
+ return _thread_service_storage;
+ }
+
+public:
+ struct OwnedLock {
+ // should be synced with ordinals of jdk.internal.vm.ThreadSnapshot.OwnedLockType enum
+ enum Type {
+ NOTHING = -1,
+ LOCKED = 0,
+ ELIMINATED = 1,
+ };
+
+ int _frame_depth;
+ Type _type;
+ // synchronization object (when type == LOCKED) or its klass (type == ELIMINATED)
+ OopHandle _obj;
+
+ OwnedLock(int depth, Type type, OopHandle obj): _frame_depth(depth), _type(type), _obj(obj) {}
+ OwnedLock(): _frame_depth(0), _type(NOTHING), _obj(nullptr) {}
+ };
+
+ struct Blocker {
+ // should be synced with ordinals of jdk.internal.vm.ThreadSnapshot.BlockerLockType enum
+ enum Type {
+ NOTHING = -1,
+ PARK_BLOCKER = 0,
+ WAITING_TO_LOCK = 1,
+ WAITING_ON = 2,
+ };
+
+ Type _type;
+ // park blocker or an object the thread waiting on/trying to lock
+ OopHandle _obj;
+
+ Blocker(Type type, OopHandle obj): _type(type), _obj(obj) {}
+ Blocker(): _type(NOTHING), _obj(nullptr) {}
+
+ bool is_empty() const {
+ return _type == NOTHING;
+ }
+ };
+
+ Handle _thread_h;
+ JavaThread* _java_thread;
+ int _frame_count; // length of _methods and _bcis arrays
+ GrowableArray* _methods;
+ GrowableArray* _bcis;
+ JavaThreadStatus _thread_status;
+ OopHandle _thread_name;
+ GrowableArray* _locks;
+ Blocker _blocker;
+
+ GetThreadSnapshotClosure(Handle thread_h, JavaThread* java_thread):
+ HandshakeClosure("GetThreadSnapshotClosure"),
+ _thread_h(thread_h), _java_thread(java_thread),
+ _frame_count(0), _methods(nullptr), _bcis(nullptr),
+ _thread_status(), _thread_name(nullptr),
+ _locks(nullptr), _blocker() {
+ }
+ virtual ~GetThreadSnapshotClosure() {
+ delete _methods;
+ delete _bcis;
+ _thread_name.release(oop_storage());
+ if (_locks != nullptr) {
+ for (int i = 0; i < _locks->length(); i++) {
+ _locks->at(i)._obj.release(oop_storage());
+ }
+ delete _locks;
+ }
+ _blocker._obj.release(oop_storage());
+ }
+
+private:
+ void detect_locks(javaVFrame* jvf, int depth) {
+ Thread* current = Thread::current();
+
+ if (depth == 0 && _blocker.is_empty()) {
+ // If this is the first frame and it is java.lang.Object.wait(...)
+ // then print out the receiver.
+ if (jvf->method()->name() == vmSymbols::wait_name() &&
+ jvf->method()->method_holder()->name() == vmSymbols::java_lang_Object()) {
+ OopHandle lock_object;
+ StackValueCollection* locs = jvf->locals();
+ if (!locs->is_empty()) {
+ StackValue* sv = locs->at(0);
+ if (sv->type() == T_OBJECT) {
+ Handle o = locs->at(0)->get_obj();
+ lock_object = OopHandle(oop_storage(), o());
+ }
+ }
+ _blocker = Blocker(Blocker::WAITING_ON, lock_object);
+ }
+ }
+
+ GrowableArray* mons = jvf->monitors();
+ if (!mons->is_empty()) {
+ for (int index = (mons->length() - 1); index >= 0; index--) {
+ MonitorInfo* monitor = mons->at(index);
+ if (monitor->eliminated() && jvf->is_compiled_frame()) { // Eliminated in compiled code
+ if (monitor->owner_is_scalar_replaced()) {
+ Klass* k = java_lang_Class::as_Klass(monitor->owner_klass());
+ _locks->push(OwnedLock(depth, OwnedLock::ELIMINATED, OopHandle(oop_storage(), k->klass_holder())));
+ } else {
+ Handle owner(current, monitor->owner());
+ if (owner.not_null()) {
+ Klass* k = owner->klass();
+ _locks->push(OwnedLock(depth, OwnedLock::ELIMINATED, OopHandle(oop_storage(), k->klass_holder())));
+ }
+ }
+ continue;
+ }
+ if (monitor->owner() != nullptr) {
+ // the monitor is associated with an object, i.e., it is locked
+
+ if (depth == 0 && _blocker.is_empty()) {
+ ObjectMonitor* pending_moninor = java_lang_VirtualThread::is_instance(_thread_h())
+ ? java_lang_VirtualThread::current_pending_monitor(_thread_h())
+ : jvf->thread()->current_pending_monitor();
+
+ markWord mark = monitor->owner()->mark();
+ // The first stage of async deflation does not affect any field
+ // used by this comparison so the ObjectMonitor* is usable here.
+ if (mark.has_monitor()) {
+ ObjectMonitor* mon = ObjectSynchronizer::read_monitor(current, monitor->owner(), mark);
+ if (// if the monitor is null we must be in the process of locking
+ mon == nullptr ||
+ // we have marked ourself as pending on this monitor
+ mon == pending_moninor ||
+ // we are not the owner of this monitor
+ (_java_thread != nullptr && !mon->is_entered(_java_thread))) {
+ _blocker = Blocker(Blocker::WAITING_TO_LOCK, OopHandle(oop_storage(), monitor->owner()));
+ continue; // go to next monitor
+ }
+ }
+ }
+ _locks->push(OwnedLock(depth, OwnedLock::LOCKED, OopHandle(oop_storage(), monitor->owner())));
+ }
+ }
+ }
+ }
+
+public:
+ void do_thread(Thread* th) override {
+ Thread* current = Thread::current();
+
+ bool is_virtual = java_lang_VirtualThread::is_instance(_thread_h());
+ if (_java_thread != nullptr) {
+ if (is_virtual) {
+ // mounted vthread, use carrier thread state
+ oop carrier_thread = java_lang_VirtualThread::carrier_thread(_thread_h());
+ _thread_status = java_lang_Thread::get_thread_status(carrier_thread);
+ } else {
+ _thread_status = java_lang_Thread::get_thread_status(_thread_h());
+ }
+ } else {
+ // unmounted vthread
+ int vt_state = java_lang_VirtualThread::state(_thread_h());
+ _thread_status = java_lang_VirtualThread::map_state_to_thread_status(vt_state);
+ }
+ _thread_name = OopHandle(oop_storage(), java_lang_Thread::name(_thread_h()));
+
+ if (_java_thread != nullptr && !_java_thread->has_last_Java_frame()) {
+ // stack trace is empty
+ return;
+ }
+
+ bool vthread_carrier = !is_virtual && (_java_thread != nullptr) && (_java_thread->vthread_continuation() != nullptr);
+
+ oop park_blocker = java_lang_Thread::park_blocker(_thread_h());
+ if (park_blocker != nullptr) {
+ _blocker = Blocker(Blocker::PARK_BLOCKER, OopHandle(oop_storage(), park_blocker));
+ }
+
+ ResourceMark rm(current);
+ HandleMark hm(current);
+
+ const int max_depth = MaxJavaStackTraceDepth;
+ const bool skip_hidden = !ShowHiddenFrames;
+
+ // Pick minimum length that will cover most cases
+ int init_length = 64;
+ _methods = new (mtInternal) GrowableArray(init_length, mtInternal);
+ _bcis = new (mtInternal) GrowableArray(init_length, mtInternal);
+ _locks = new (mtInternal) GrowableArray(init_length, mtInternal);
+ int total_count = 0;
+
+ vframeStream vfst(_java_thread != nullptr
+ ? vframeStream(_java_thread, false, true, vthread_carrier)
+ : vframeStream(java_lang_VirtualThread::continuation(_thread_h())));
+
+ for (;
+ !vfst.at_end() && (max_depth == 0 || max_depth != total_count);
+ vfst.next()) {
+
+ detect_locks(vfst.asJavaVFrame(), total_count);
+
+ if (skip_hidden && (vfst.method()->is_hidden() ||
+ vfst.method()->is_continuation_enter_intrinsic())) {
+ continue;
+ }
+ _methods->push(vfst.method());
+ _bcis->push(vfst.bci());
+ total_count++;
+ }
+
+ _frame_count = total_count;
+ }
+};
+
+class jdk_internal_vm_ThreadLock: AllStatic {
+ static bool _inited;
+ static int _depth_offset;
+ static int _typeOrdinal_offset;
+ static int _obj_offset;
+
+ static void compute_offsets(InstanceKlass* klass, TRAPS) {
+ JavaClasses::compute_offset(_depth_offset, klass, "depth", vmSymbols::int_signature(), false);
+ JavaClasses::compute_offset(_typeOrdinal_offset, klass, "typeOrdinal", vmSymbols::int_signature(), false);
+ JavaClasses::compute_offset(_obj_offset, klass, "obj", vmSymbols::object_signature(), false);
+ }
+public:
+ static void init(InstanceKlass* klass, TRAPS) {
+ if (!_inited) {
+ compute_offsets(klass, CHECK);
+ _inited = true;
+ }
+ }
+
+ static Handle create(InstanceKlass* klass, int depth, int type_ordinal, OopHandle obj, TRAPS) {
+ init(klass, CHECK_NH);
+ Handle result = klass->allocate_instance_handle(CHECK_NH);
+ result->int_field_put(_depth_offset, depth);
+ result->int_field_put(_typeOrdinal_offset, type_ordinal);
+ result->obj_field_put(_obj_offset, obj.resolve());
+ return result;
+ }
+};
+
+bool jdk_internal_vm_ThreadLock::_inited = false;
+int jdk_internal_vm_ThreadLock::_depth_offset;
+int jdk_internal_vm_ThreadLock::_typeOrdinal_offset;
+int jdk_internal_vm_ThreadLock::_obj_offset;
+
+class jdk_internal_vm_ThreadSnapshot: AllStatic {
+ static bool _inited;
+ static int _name_offset;
+ static int _threadStatus_offset;
+ static int _carrierThread_offset;
+ static int _stackTrace_offset;
+ static int _locks_offset;
+ static int _blockerTypeOrdinal_offset;
+ static int _blockerObject_offset;
+
+ static void compute_offsets(InstanceKlass* klass, TRAPS) {
+ JavaClasses::compute_offset(_name_offset, klass, "name", vmSymbols::string_signature(), false);
+ JavaClasses::compute_offset(_threadStatus_offset, klass, "threadStatus", vmSymbols::int_signature(), false);
+ JavaClasses::compute_offset(_carrierThread_offset, klass, "carrierThread", vmSymbols::thread_signature(), false);
+ JavaClasses::compute_offset(_stackTrace_offset, klass, "stackTrace", vmSymbols::java_lang_StackTraceElement_array(), false);
+ JavaClasses::compute_offset(_locks_offset, klass, "locks", vmSymbols::jdk_internal_vm_ThreadLock_array(), false);
+ JavaClasses::compute_offset(_blockerTypeOrdinal_offset, klass, "blockerTypeOrdinal", vmSymbols::int_signature(), false);
+ JavaClasses::compute_offset(_blockerObject_offset, klass, "blockerObject", vmSymbols::object_signature(), false);
+ }
+public:
+ static void init(InstanceKlass* klass, TRAPS) {
+ if (!_inited) {
+ compute_offsets(klass, CHECK);
+ _inited = true;
+ }
+ }
+
+ static Handle allocate(InstanceKlass* klass, TRAPS) {
+ init(klass, CHECK_NH);
+ Handle h_k = klass->allocate_instance_handle(CHECK_NH);
+ return h_k;
+ }
+
+ static void set_name(oop snapshot, oop name) {
+ snapshot->obj_field_put(_name_offset, name);
+ }
+ static void set_thread_status(oop snapshot, int status) {
+ snapshot->int_field_put(_threadStatus_offset, status);
+ }
+ static void set_carrier_thread(oop snapshot, oop carrier_thread) {
+ snapshot->obj_field_put(_carrierThread_offset, carrier_thread);
+ }
+ static void set_stack_trace(oop snapshot, oop trace) {
+ snapshot->obj_field_put(_stackTrace_offset, trace);
+ }
+ static void set_locks(oop snapshot, oop locks) {
+ snapshot->obj_field_put(_locks_offset, locks);
+ }
+ static void set_blocker(oop snapshot, int type_ordinal, oop lock) {
+ snapshot->int_field_put(_blockerTypeOrdinal_offset, type_ordinal);
+ snapshot->obj_field_put(_blockerObject_offset, lock);
+ }
+};
+
+bool jdk_internal_vm_ThreadSnapshot::_inited = false;
+int jdk_internal_vm_ThreadSnapshot::_name_offset;
+int jdk_internal_vm_ThreadSnapshot::_threadStatus_offset;
+int jdk_internal_vm_ThreadSnapshot::_carrierThread_offset;
+int jdk_internal_vm_ThreadSnapshot::_stackTrace_offset;
+int jdk_internal_vm_ThreadSnapshot::_locks_offset;
+int jdk_internal_vm_ThreadSnapshot::_blockerTypeOrdinal_offset;
+int jdk_internal_vm_ThreadSnapshot::_blockerObject_offset;
+
+oop ThreadSnapshotFactory::get_thread_snapshot(jobject jthread, TRAPS) {
+ ThreadsListHandle tlh(THREAD);
+
+ ResourceMark rm(THREAD);
+ HandleMark hm(THREAD);
+ Handle thread_h(THREAD, JNIHandles::resolve(jthread));
+
+ // wrapper to auto delete JvmtiVTMSTransitionDisabler
+ class TransitionDisabler {
+ JvmtiVTMSTransitionDisabler* _transition_disabler;
+ public:
+ TransitionDisabler(): _transition_disabler(nullptr) {}
+ ~TransitionDisabler() {
+ reset();
+ }
+ void init(jobject jthread) {
+ _transition_disabler = new (mtInternal) JvmtiVTMSTransitionDisabler(jthread);
+ }
+ void reset() {
+ if (_transition_disabler != nullptr) {
+ delete _transition_disabler;
+ _transition_disabler = nullptr;
+ }
+ }
+ } transition_disabler;
+
+ JavaThread* java_thread = nullptr;
+ bool is_virtual = java_lang_VirtualThread::is_instance(thread_h());
+ Handle carrier_thread;
+ if (is_virtual) {
+ // 1st need to disable mount/unmount transitions
+ transition_disabler.init(jthread);
+
+ carrier_thread = Handle(THREAD, java_lang_VirtualThread::carrier_thread(thread_h()));
+ if (carrier_thread != nullptr) {
+ java_thread = java_lang_Thread::thread(carrier_thread());
+ }
+ } else {
+ java_thread = java_lang_Thread::thread(thread_h());
+ }
+
+ // Handshake with target
+ GetThreadSnapshotClosure cl(thread_h, java_thread);
+ if (java_thread == nullptr) {
+ // unmounted vthread, execute on the current thread
+ cl.do_thread(nullptr);
+ } else {
+ Handshake::execute(&cl, &tlh, java_thread);
+ }
+
+ // all info is collected, can enable transitions.
+ transition_disabler.reset();
+
+ // StackTrace
+ InstanceKlass* ste_klass = vmClasses::StackTraceElement_klass();
+ assert(ste_klass != nullptr, "must be loaded");
+
+ objArrayHandle trace = oopFactory::new_objArray_handle(ste_klass, cl._frame_count, CHECK_NULL);
+
+ for (int i = 0; i < cl._frame_count; i++) {
+ methodHandle method(THREAD, cl._methods->at(i));
+ oop element = java_lang_StackTraceElement::create(method, cl._bcis->at(i), CHECK_NULL);
+ trace->obj_at_put(i, element);
+ }
+
+ // Locks
+ Symbol* lock_sym = vmSymbols::jdk_internal_vm_ThreadLock();
+ Klass* lock_k = SystemDictionary::resolve_or_fail(lock_sym, true, CHECK_NULL);
+ InstanceKlass* lock_klass = InstanceKlass::cast(lock_k);
+
+ objArrayHandle locks;
+ if (cl._locks != nullptr && cl._locks->length() > 0) {
+ locks = oopFactory::new_objArray_handle(lock_klass, cl._locks->length(), CHECK_NULL);
+ for (int n = 0; n < cl._locks->length(); n++) {
+ GetThreadSnapshotClosure::OwnedLock* lock_info = cl._locks->adr_at(n);
+
+ Handle lock = jdk_internal_vm_ThreadLock::create(lock_klass,
+ lock_info->_frame_depth, lock_info->_type, lock_info->_obj, CHECK_NULL);
+ locks->obj_at_put(n, lock());
+ }
+ }
+
+ // call static StackTraceElement[] StackTraceElement.of(StackTraceElement[] stackTrace)
+ // to properly initialize STEs.
+ JavaValue result(T_OBJECT);
+ JavaCalls::call_static(&result,
+ ste_klass,
+ vmSymbols::java_lang_StackTraceElement_of_name(),
+ vmSymbols::java_lang_StackTraceElement_of_signature(),
+ trace,
+ CHECK_NULL);
+ // the method return the same trace array
+
+ Symbol* snapshot_klass_name = vmSymbols::jdk_internal_vm_ThreadSnapshot();
+ Klass* snapshot_klass = SystemDictionary::resolve_or_fail(snapshot_klass_name, true, CHECK_NULL);
+ if (snapshot_klass->should_be_initialized()) {
+ snapshot_klass->initialize(CHECK_NULL);
+ }
+
+ Handle snapshot = jdk_internal_vm_ThreadSnapshot::allocate(InstanceKlass::cast(snapshot_klass), CHECK_NULL);
+ jdk_internal_vm_ThreadSnapshot::set_name(snapshot(), cl._thread_name.resolve());
+ jdk_internal_vm_ThreadSnapshot::set_thread_status(snapshot(), (int)cl._thread_status);
+ jdk_internal_vm_ThreadSnapshot::set_carrier_thread(snapshot(), carrier_thread());
+ jdk_internal_vm_ThreadSnapshot::set_stack_trace(snapshot(), trace());
+ jdk_internal_vm_ThreadSnapshot::set_locks(snapshot(), locks());
+ if (!cl._blocker.is_empty()) {
+ jdk_internal_vm_ThreadSnapshot::set_blocker(snapshot(), cl._blocker._type, cl._blocker._obj.resolve());
+ }
+ return snapshot();
+}
+
+#endif // INCLUDE_JVMTI
+
diff --git a/src/hotspot/share/services/threadService.hpp b/src/hotspot/share/services/threadService.hpp
index 17d8ba9b3cc..b1de4bc8703 100644
--- a/src/hotspot/share/services/threadService.hpp
+++ b/src/hotspot/share/services/threadService.hpp
@@ -631,4 +631,11 @@ class JavaThreadSleepState : public JavaThreadStatusChanger {
}
};
+
+// jdk.internal.vm.ThreadSnapshot support
+class ThreadSnapshotFactory: AllStatic {
+public:
+ JVMTI_ONLY(static oop get_thread_snapshot(jobject jthread, TRAPS);)
+};
+
#endif // SHARE_SERVICES_THREADSERVICE_HPP
diff --git a/src/hotspot/share/utilities/enumIterator.hpp b/src/hotspot/share/utilities/enumIterator.hpp
index 6fa7f071f4a..42cc405fcd7 100644
--- a/src/hotspot/share/utilities/enumIterator.hpp
+++ b/src/hotspot/share/utilities/enumIterator.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,13 +25,14 @@
#ifndef SHARE_UTILITIES_ENUMITERATOR_HPP
#define SHARE_UTILITIES_ENUMITERATOR_HPP
-#include
-#include
#include "memory/allStatic.hpp"
#include "metaprogramming/enableIf.hpp"
#include "metaprogramming/primitiveConversions.hpp"
#include "utilities/debug.hpp"
+#include
+#include
+
// Iteration support for enums.
//
// E is enum type, U is underlying type of E.
@@ -147,6 +148,12 @@ public:
assert(value <= end, "out of range");
}
+ template
+ static constexpr void assert_in_range() {
+ static_assert(_start <= static_cast(Value), "out of range");
+ static_assert(static_cast(Value) <= _end, "out of range");
+ }
+
// Convert an enumerator value to the corresponding underlying type.
static constexpr Underlying underlying_value(T value) {
return static_cast(value);
@@ -229,6 +236,12 @@ class EnumRange {
assert(size() > 0, "empty range");
}
+ struct ConstExprConstructTag {};
+
+ constexpr EnumRange(T start, T end, ConstExprConstructTag) :
+ _start(Traits::underlying_value(start)),
+ _end(Traits::underlying_value(end)) {}
+
public:
using EnumType = T;
using Iterator = EnumIterator;
@@ -252,6 +265,14 @@ public:
assert(start <= end, "invalid range");
}
+ template