diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk index 0a897230f83..ffea9aa3926 100644 --- a/make/hotspot/lib/JvmFeatures.gmk +++ b/make/hotspot/lib/JvmFeatures.gmk @@ -125,6 +125,7 @@ endif ifneq ($(call check-jvm-feature, cds), true) JVM_CFLAGS_FEATURES += -DINCLUDE_CDS=0 JVM_EXCLUDE_FILES += \ + aotCodeCache.cpp \ classLoaderDataShared.cpp \ classLoaderExt.cpp \ systemDictionaryShared.cpp diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index fd6af0b9b4b..cf347768de3 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -675,6 +675,9 @@ void MacroAssembler::set_last_Java_frame(Register last_java_sp, } static inline bool target_needs_far_branch(address addr) { + if (AOTCodeCache::is_on_for_dump()) { + return true; + } // codecache size <= 128M if (!MacroAssembler::far_branches()) { return false; @@ -859,6 +862,9 @@ void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, in // Check the entry target is always reachable from any branch. static bool is_always_within_branch_range(Address entry) { + if (AOTCodeCache::is_on_for_dump()) { + return false; + } const address target = entry.target(); if (!CodeCache::contains(target)) { @@ -2154,7 +2160,7 @@ void MacroAssembler::call_VM_leaf_base(address entry_point, stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize))); - mov(rscratch1, entry_point); + mov(rscratch1, RuntimeAddress(entry_point)); blr(rscratch1); if (retaddr) bind(*retaddr); @@ -3231,9 +3237,13 @@ void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Regis } void MacroAssembler::stop(const char* msg) { - BLOCK_COMMENT(msg); + // Skip AOT caching C strings in scratch buffer. + const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg); + BLOCK_COMMENT(str); + // load msg into r0 so we can access it from the signal handler + // ExternalAddress enables saving and restoring via the code cache + lea(c_rarg0, ExternalAddress((address) str)); dcps1(0xdeae); - emit_int64((uintptr_t)msg); } void MacroAssembler::unimplemented(const char* what) { diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index e6075b0073b..17ee72a00c0 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -27,6 +27,7 @@ #define CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP #include "asm/assembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "code/vmreg.hpp" #include "metaprogramming/enableIf.hpp" #include "oops/compressedOops.hpp" @@ -1315,6 +1316,10 @@ public: // Check if branches to the non nmethod section require a far jump static bool codestub_branch_needs_far_jump() { + if (AOTCodeCache::is_on_for_dump()) { + // To calculate far_codestub_branch_size correctly. + return true; + } return CodeCache::max_distance_to_non_nmethod() > branch_range; } diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index 967984b8821..0c3dfabc93e 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -557,40 +557,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // If this happens, control eventually transfers back to the compiled // caller, but with an uncorrected stack, causing delayed havoc. - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { -#if 0 - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, r11, - Interpreter::code()->code_start(), Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); -#endif - } - // Cut-out for having no stack args. int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord; if (comp_args_on_stack) { @@ -711,12 +677,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -777,7 +743,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } static int c_calling_convention_priv(const BasicType *sig_bt, diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp index 6dde82daaf9..8ba847e7e32 100644 --- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp +++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp @@ -612,12 +612,12 @@ static void gen_c2i_adapter(MacroAssembler *masm, } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -637,7 +637,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm address c2i_entry = __ pc(); gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr); + return; } diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index 1c9c88b3c30..5a33a14f79e 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -1143,12 +1143,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, __ bctr(); } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry; address c2i_unverified_entry; address c2i_entry; @@ -1223,8 +1223,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, call_interpreter, ientry); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, - c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } // An oop arg. Must pass a handle not the oop itself. diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp index 10790841490..5f53485a97f 100644 --- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp +++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp @@ -596,12 +596,13 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { + +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -658,7 +659,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } int SharedRuntime::vector_calling_convention(VMRegPair *regs, diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index 099e28a3adc..bd5bbf4c7e5 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -2352,12 +2352,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, __ z_br(Z_R1_scratch); } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { __ align(CodeEntryAlignment); address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -2411,7 +2411,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } // This function returns the adjust size (in number of words) to a c2i adapter diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 4d3bd12ed66..35e461b601f 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -24,6 +24,7 @@ #include "asm/assembler.hpp" #include "asm/assembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "code/compiledIC.hpp" #include "compiler/compiler_globals.hpp" #include "compiler/disassembler.hpp" @@ -366,7 +367,9 @@ void MacroAssembler::stop(const char* msg) { lea(c_rarg1, InternalAddress(rip)); movq(c_rarg2, rsp); // pass pointer to regs array } - lea(c_rarg0, ExternalAddress((address) msg)); + // Skip AOT caching C strings in scratch buffer. + const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg); + lea(c_rarg0, ExternalAddress((address) str)); andq(rsp, -16); // align stack as required by ABI call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); hlt(); diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 621340964ac..7811d59d12d 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -675,7 +675,6 @@ static void patch_callers_callsite(MacroAssembler *masm) { __ bind(L); } - static void gen_c2i_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -826,19 +825,6 @@ static void gen_c2i_adapter(MacroAssembler *masm, __ jmp(rcx); } -static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, - address code_start, address code_end, - Label& L_ok) { - Label L_fail; - __ lea(temp_reg, AddressLiteral(code_start, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::belowEqual, L_fail); - __ lea(temp_reg, AddressLiteral(code_end, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::below, L_ok); - __ bind(L_fail); -} - void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -871,41 +857,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // If this happens, control eventually transfers back to the compiled // caller, but with an uncorrected stack, causing delayed havoc. - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - // Pick up the return address - __ movptr(rax, Address(rsp, 0)); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, r11, - Interpreter::code()->code_start(), - Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); - } - // Must preserve original SP for loading incoming arguments because // we need to align the outgoing SP for compiled code. __ movptr(r11, rsp); @@ -1050,12 +1001,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -1117,7 +1068,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } int SharedRuntime::c_calling_convention(const BasicType *sig_bt, diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp index f141135ff95..60a873ab31f 100644 --- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp +++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp @@ -50,18 +50,17 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt, return 0; } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters( - MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint *fingerprint) { - return AdapterHandlerLibrary::new_entry( - fingerprint, - CAST_FROM_FN_PTR(address,zero_null_code_stub), - CAST_FROM_FN_PTR(address,zero_null_code_stub), - CAST_FROM_FN_PTR(address,zero_null_code_stub)); +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { + handler->set_entry_points(CAST_FROM_FN_PTR(address,zero_null_code_stub), + CAST_FROM_FN_PTR(address,zero_null_code_stub), + CAST_FROM_FN_PTR(address,zero_null_code_stub), + nullptr); + return; } nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp index 7b35317882d..aeba308d3a2 100644 --- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp +++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp @@ -271,11 +271,8 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, stub = SharedRuntime::handle_unsafe_access(thread, next_pc); } } else if (sig == SIGILL && nativeInstruction_at(pc)->is_stop()) { - // Pull a pointer to the error message out of the instruction - // stream. - const uint64_t *detail_msg_ptr - = (uint64_t*)(pc + NativeInstruction::instruction_size); - const char *detail_msg = (const char *)*detail_msg_ptr; + // A pointer to the message will have been placed in r0 + const char *detail_msg = (const char *)(uc->uc_mcontext->DU3_PREFIX(ss,x[0])); const char *msg = "stop"; if (TraceTraps) { tty->print_cr("trap: %s: (SIGILL)", msg); diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp index 7728c62682c..4335733d1fe 100644 --- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp @@ -248,11 +248,8 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, stub = SharedRuntime::handle_unsafe_access(thread, next_pc); } } else if (sig == SIGILL && nativeInstruction_at(pc)->is_stop()) { - // Pull a pointer to the error message out of the instruction - // stream. - const uint64_t *detail_msg_ptr - = (uint64_t*)(pc + NativeInstruction::instruction_size); - const char *detail_msg = (const char *)*detail_msg_ptr; + // A pointer to the message will have been placed in r0 + const char *detail_msg = (const char *)(uc->uc_mcontext.regs[0]); const char *msg = "stop"; if (TraceTraps) { tty->print_cr("trap: %s: (SIGILL)", msg); diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp index fc8f72a881e..be7a39380d9 100644 --- a/src/hotspot/share/asm/codeBuffer.cpp +++ b/src/hotspot/share/asm/codeBuffer.cpp @@ -722,7 +722,7 @@ void CodeBuffer::copy_code_to(CodeBlob* dest_blob) { #ifndef PRODUCT if (PrintNMethods && (WizardMode || Verbose)) { tty->print("done with CodeBuffer:"); - ((CodeBuffer*)this)->print(); + ((CodeBuffer*)this)->print_on(tty); } #endif //PRODUCT @@ -861,7 +861,7 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) { #ifndef PRODUCT if (PrintNMethods && (WizardMode || Verbose)) { tty->print("expanding CodeBuffer:"); - this->print(); + this->print_on(tty); } if (StressCodeBuffers && blob() != nullptr) { @@ -949,7 +949,7 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) { _decode_begin = nullptr; // sanity if (PrintNMethods && (WizardMode || Verbose)) { tty->print("expanded CodeBuffer:"); - this->print(); + this->print_on(tty); } #endif //PRODUCT } @@ -1066,24 +1066,24 @@ void CodeBuffer::decode() { _decode_begin = insts_end(); } -void CodeSection::print(const char* name) { +void CodeSection::print_on(outputStream* st, const char* name) { csize_t locs_size = locs_end() - locs_start(); - tty->print_cr(" %7s.code = " PTR_FORMAT " : " PTR_FORMAT " : " PTR_FORMAT " (%d of %d)", + st->print_cr(" %7s.code = " PTR_FORMAT " : " PTR_FORMAT " : " PTR_FORMAT " (%d of %d)", name, p2i(start()), p2i(end()), p2i(limit()), size(), capacity()); - tty->print_cr(" %7s.locs = " PTR_FORMAT " : " PTR_FORMAT " : " PTR_FORMAT " (%d of %d) point=%d", + st->print_cr(" %7s.locs = " PTR_FORMAT " : " PTR_FORMAT " : " PTR_FORMAT " (%d of %d) point=%d", name, p2i(locs_start()), p2i(locs_end()), p2i(locs_limit()), locs_size, locs_capacity(), locs_point_off()); if (PrintRelocations && (locs_size != 0)) { RelocIterator iter(this); - iter.print(); + iter.print_on(st); } } -void CodeBuffer::print() { - tty->print_cr("CodeBuffer:"); +void CodeBuffer::print_on(outputStream* st) { + st->print_cr("CodeBuffer:%s", name()); for (int n = 0; n < (int)SECT_LIMIT; n++) { // print each section CodeSection* cs = code_section(n); - cs->print(code_section_name(n)); + cs->print_on(st, code_section_name(n)); } } diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp index b38cc74cc3b..95cd4b7f912 100644 --- a/src/hotspot/share/asm/codeBuffer.hpp +++ b/src/hotspot/share/asm/codeBuffer.hpp @@ -89,6 +89,7 @@ public: // They are filled concurrently, and concatenated at the end. class CodeSection { friend class CodeBuffer; + friend class AOTCodeReader; public: typedef int csize_t; // code size type; would be size_t except for history @@ -283,7 +284,7 @@ class CodeSection { #ifndef PRODUCT void decode(); - void print(const char* name); + void print_on(outputStream* st, const char* name); #endif //PRODUCT }; @@ -386,6 +387,7 @@ typedef GrowableArray SharedStubToInterpRequests; class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) { friend class CodeSection; friend class StubCodeGenerator; + friend class AOTCodeReader; private: // CodeBuffers must be allocated on the stack except for a single @@ -742,7 +744,7 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) { // Printing / Decoding // decodes from decode_begin() to code_end() and sets decode_begin to end void decode(); - void print(); + void print_on(outputStream* st); #endif // Directly disassemble code buffer. void decode(address start, address end); diff --git a/src/hotspot/share/cds/aotCacheAccess.cpp b/src/hotspot/share/cds/aotCacheAccess.cpp new file mode 100644 index 00000000000..b6c4a201da5 --- /dev/null +++ b/src/hotspot/share/cds/aotCacheAccess.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "cds/aotCacheAccess.hpp" +#include "cds/archiveBuilder.hpp" +#include "cds/cdsConfig.hpp" +#include "cds/filemap.hpp" +#include "cds/heapShared.hpp" +#include "cds/metaspaceShared.hpp" +#include "classfile/stringTable.hpp" +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "memory/resourceArea.hpp" +#include "memory/universe.hpp" +#include "memory/virtualspace.hpp" +#include "oops/instanceKlass.hpp" + +void* AOTCacheAccess::allocate_aot_code_region(size_t size) { + assert(CDSConfig::is_dumping_final_static_archive(), "must be"); + return (void*)ArchiveBuilder::ac_region_alloc(size); +} + +size_t AOTCacheAccess::get_aot_code_region_size() { + assert(CDSConfig::is_using_archive(), "must be"); + FileMapInfo* mapinfo = FileMapInfo::current_info(); + assert(mapinfo != nullptr, "must be"); + return mapinfo->region_at(MetaspaceShared::ac)->used_aligned(); +} + +bool AOTCacheAccess::map_aot_code_region(ReservedSpace rs) { + FileMapInfo* static_mapinfo = FileMapInfo::current_info(); + assert(UseSharedSpaces && static_mapinfo != nullptr, "must be"); + return static_mapinfo->map_aot_code_region(rs); +} diff --git a/src/hotspot/share/cds/aotCacheAccess.hpp b/src/hotspot/share/cds/aotCacheAccess.hpp new file mode 100644 index 00000000000..31fb9da534d --- /dev/null +++ b/src/hotspot/share/cds/aotCacheAccess.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_CDS_AOTCACHEACCESS_HPP +#define SHARE_CDS_AOTCACHEACCESS_HPP + +#include "cds/archiveBuilder.hpp" +#include "cds/archiveUtils.hpp" +#include "memory/allStatic.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/globalDefinitions.hpp" + +class ReservedSpace; + +// AOT Cache API for AOT compiler + +class AOTCacheAccess : AllStatic { +public: + static void* allocate_aot_code_region(size_t size) NOT_CDS_RETURN_(nullptr); + + static size_t get_aot_code_region_size() NOT_CDS_RETURN_(0); + + static bool map_aot_code_region(ReservedSpace rs) NOT_CDS_RETURN_(false); +}; + +#endif // SHARE_CDS_AOTCACHEACCESS_HPP diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index ef75bd42323..3de4cdec187 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -42,6 +42,7 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionaryShared.hpp" #include "classfile/vmClasses.hpp" +#include "code/aotCodeCache.hpp" #include "interpreter/abstractInterpreter.hpp" #include "jvm.h" #include "logging/log.hpp" @@ -163,6 +164,7 @@ ArchiveBuilder::ArchiveBuilder() : _pz_region("pz", MAX_SHARED_DELTA), // protection zone -- used only during dumping; does NOT exist in cds archive. _rw_region("rw", MAX_SHARED_DELTA), _ro_region("ro", MAX_SHARED_DELTA), + _ac_region("ac", MAX_SHARED_DELTA), _ptrmap(mtClassShared), _rw_ptrmap(mtClassShared), _ro_ptrmap(mtClassShared), @@ -306,7 +308,8 @@ void ArchiveBuilder::sort_klasses() { } address ArchiveBuilder::reserve_buffer() { - size_t buffer_size = LP64_ONLY(CompressedClassSpaceSize) NOT_LP64(256 * M); + // AOTCodeCache::max_aot_code_size() accounts for aot code region. + size_t buffer_size = LP64_ONLY(CompressedClassSpaceSize) NOT_LP64(256 * M) + AOTCodeCache::max_aot_code_size(); ReservedSpace rs = MemoryReserver::reserve(buffer_size, MetaspaceShared::core_region_alignment(), os::vm_page_size(), @@ -532,6 +535,13 @@ ArchiveBuilder::FollowMode ArchiveBuilder::get_follow_mode(MetaspaceClosure::Ref } else if (ref->msotype() == MetaspaceObj::MethodDataType || ref->msotype() == MetaspaceObj::MethodCountersType) { return set_to_null; + } else if (ref->msotype() == MetaspaceObj::AdapterHandlerEntryType) { + if (AOTCodeCache::is_dumping_adapters()) { + AdapterHandlerEntry* entry = (AdapterHandlerEntry*)ref->obj(); + return AdapterHandlerLibrary::is_abstract_method_adapter(entry) ? set_to_null : make_a_copy; + } else { + return set_to_null; + } } else { if (ref->msotype() == MetaspaceObj::ClassType) { Klass* klass = (Klass*)ref->obj(); @@ -705,6 +715,11 @@ void ArchiveBuilder::mark_and_relocate_to_buffered_addr(address* ptr_location) { ArchivePtrMarker::mark_pointer(ptr_location); } +bool ArchiveBuilder::has_been_archived(address src_addr) const { + SourceObjInfo* p = _src_obj_table.get(src_addr); + return (p != nullptr); +} + bool ArchiveBuilder::has_been_buffered(address src_addr) const { if (RegeneratedClasses::has_been_regenerated(src_addr) || _src_obj_table.get(src_addr) == nullptr || @@ -973,6 +988,15 @@ address ArchiveBuilder::offset_to_buffered_address(u4 offset) const { return buffered_addr; } +void ArchiveBuilder::start_ac_region() { + ro_region()->pack(); + start_dump_region(&_ac_region); +} + +void ArchiveBuilder::end_ac_region() { + _ac_region.pack(); +} + #if INCLUDE_CDS_JAVA_HEAP narrowKlass ArchiveBuilder::get_requested_narrow_klass(Klass* k) { assert(CDSConfig::is_dumping_heap(), "sanity"); @@ -1082,8 +1106,9 @@ int ArchiveBuilder::precomputed_narrow_klass_shift() { #endif // _LP64 void ArchiveBuilder::relocate_to_requested() { - ro_region()->pack(); - + if (!ro_region()->is_packed()) { + ro_region()->pack(); + } size_t my_archive_size = buffer_top() - buffer_bottom(); if (CDSConfig::is_dumping_static_archive()) { @@ -1527,6 +1552,7 @@ void ArchiveBuilder::write_archive(FileMapInfo* mapinfo, ArchiveHeapInfo* heap_i write_region(mapinfo, MetaspaceShared::rw, &_rw_region, /*read_only=*/false,/*allow_exec=*/false); write_region(mapinfo, MetaspaceShared::ro, &_ro_region, /*read_only=*/true, /*allow_exec=*/false); + write_region(mapinfo, MetaspaceShared::ac, &_ac_region, /*read_only=*/false,/*allow_exec=*/false); // Split pointer map into read-write and read-only bitmaps ArchivePtrMarker::initialize_rw_ro_maps(&_rw_ptrmap, &_ro_ptrmap); @@ -1579,6 +1605,7 @@ void ArchiveBuilder::print_region_stats(FileMapInfo *mapinfo, ArchiveHeapInfo* h _rw_region.print(total_reserved); _ro_region.print(total_reserved); + _ac_region.print(total_reserved); print_bitmap_region_stats(bitmap_used, total_reserved); diff --git a/src/hotspot/share/cds/archiveBuilder.hpp b/src/hotspot/share/cds/archiveBuilder.hpp index 5913ae29c78..531c43e1cbf 100644 --- a/src/hotspot/share/cds/archiveBuilder.hpp +++ b/src/hotspot/share/cds/archiveBuilder.hpp @@ -217,6 +217,7 @@ private: DumpRegion _rw_region; DumpRegion _ro_region; + DumpRegion _ac_region; // AOT code // Combined bitmap to track pointers in both RW and RO regions. This is updated // as objects are copied into RW and RO. @@ -372,6 +373,7 @@ public: DumpRegion* pz_region() { return &_pz_region; } DumpRegion* rw_region() { return &_rw_region; } DumpRegion* ro_region() { return &_ro_region; } + DumpRegion* ac_region() { return &_ac_region; } static char* rw_region_alloc(size_t num_bytes) { return current()->rw_region()->allocate(num_bytes); @@ -379,6 +381,12 @@ public: static char* ro_region_alloc(size_t num_bytes) { return current()->ro_region()->allocate(num_bytes); } + static char* ac_region_alloc(size_t num_bytes) { + return current()->ac_region()->allocate(num_bytes); + } + + void start_ac_region(); + void end_ac_region(); template static Array* new_ro_array(int length) { @@ -426,6 +434,8 @@ public: mark_and_relocate_to_buffered_addr((address*)ptr_location); } + bool has_been_archived(address src_addr) const; + bool has_been_buffered(address src_addr) const; template bool has_been_buffered(T src_addr) const { return has_been_buffered((address)src_addr); diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp index 217b30e401d..0b01494574c 100644 --- a/src/hotspot/share/cds/archiveUtils.cpp +++ b/src/hotspot/share/cds/archiveUtils.cpp @@ -270,9 +270,10 @@ void DumpRegion::append_intptr_t(intptr_t n, bool need_to_mark) { } void DumpRegion::print(size_t total_bytes) const { + char* base = used() > 0 ? ArchiveBuilder::current()->to_requested(_base) : nullptr; log_debug(cds)("%s space: %9zu [ %4.1f%% of total] out of %9zu bytes [%5.1f%% used] at " INTPTR_FORMAT, _name, used(), percent_of(used(), total_bytes), reserved(), percent_of(used(), reserved()), - p2i(ArchiveBuilder::current()->to_requested(_base))); + p2i(base)); } void DumpRegion::print_out_of_space_msg(const char* failing_region, size_t needed_bytes) { @@ -295,7 +296,10 @@ void DumpRegion::init(ReservedSpace* rs, VirtualSpace* vs) { } void DumpRegion::pack(DumpRegion* next) { - assert(!is_packed(), "sanity"); + if (!is_packed()) { + _end = (char*)align_up(_top, MetaspaceShared::core_region_alignment()); + _is_packed = true; + } _end = (char*)align_up(_top, MetaspaceShared::core_region_alignment()); _is_packed = true; if (next != nullptr) { diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp index 56d28ec910c..836ec93f641 100644 --- a/src/hotspot/share/cds/archiveUtils.hpp +++ b/src/hotspot/share/cds/archiveUtils.hpp @@ -180,6 +180,7 @@ public: bool is_allocatable() const { return !is_packed() && _base != nullptr; } + bool is_empty() const { return _base == _top; } void print(size_t total_bytes) const; void print_out_of_space_msg(const char* failing_region, size_t needed_bytes); diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp index 9af83653351..701b2584774 100644 --- a/src/hotspot/share/cds/cdsConfig.cpp +++ b/src/hotspot/share/cds/cdsConfig.cpp @@ -402,6 +402,10 @@ void CDSConfig::check_aot_flags() { CHECK_SINGLE_PATH(AOTCache); CHECK_SINGLE_PATH(AOTConfiguration); + if (FLAG_IS_DEFAULT(AOTCache) && AOTAdapterCaching) { + log_debug(aot,codecache,init)("AOTCache is not specified - AOTAdapterCaching is ignored"); + } + if (FLAG_IS_DEFAULT(AOTCache) && FLAG_IS_DEFAULT(AOTConfiguration) && FLAG_IS_DEFAULT(AOTMode)) { // AOTCache/AOTConfiguration/AOTMode not used. return; @@ -517,7 +521,7 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla } if (is_dumping_static_archive()) { - if (is_dumping_preimage_static_archive()) { + if (is_dumping_preimage_static_archive() || is_dumping_final_static_archive()) { // Don't tweak execution mode } else if (!mode_flag_cmd_line) { // By default, -Xshare:dump runs in interpreter-only mode, which is required for deterministic archive. @@ -844,3 +848,22 @@ bool CDSConfig::is_dumping_method_handles() { } #endif // INCLUDE_CDS_JAVA_HEAP + +// AOT code generation and its archiving is disabled by default. +// We enable it only in the final image dump after the metadata and heap are dumped. +// This affects only JITed code because it may have embedded oops and metadata pointers +// which AOT code encodes as offsets in final CDS archive regions. + +static bool _is_dumping_aot_code = false; + +bool CDSConfig::is_dumping_aot_code() { + return _is_dumping_aot_code; +} + +void CDSConfig::disable_dumping_aot_code() { + _is_dumping_aot_code = false; +} + +void CDSConfig::enable_dumping_aot_code() { + _is_dumping_aot_code = true; +} diff --git a/src/hotspot/share/cds/cdsConfig.hpp b/src/hotspot/share/cds/cdsConfig.hpp index e96291f6534..e82acebd175 100644 --- a/src/hotspot/share/cds/cdsConfig.hpp +++ b/src/hotspot/share/cds/cdsConfig.hpp @@ -181,6 +181,12 @@ public: static void stop_dumping_full_module_graph(const char* reason = nullptr) NOT_CDS_JAVA_HEAP_RETURN; static void stop_using_full_module_graph(const char* reason = nullptr) NOT_CDS_JAVA_HEAP_RETURN; + // --- AOT code + + static bool is_dumping_aot_code() NOT_CDS_RETURN_(false); + static void disable_dumping_aot_code() NOT_CDS_RETURN; + static void enable_dumping_aot_code() NOT_CDS_RETURN; + // Some CDS functions assume that they are called only within a single-threaded context. I.e., // they are called from: // - The VM thread (e.g., inside VM_PopulateDumpSharedSpace) diff --git a/src/hotspot/share/cds/cds_globals.hpp b/src/hotspot/share/cds/cds_globals.hpp index 2dae9b45221..26853a3fb74 100644 --- a/src/hotspot/share/cds/cds_globals.hpp +++ b/src/hotspot/share/cds/cds_globals.hpp @@ -127,6 +127,22 @@ product(bool, AOTCacheParallelRelocation, true, DIAGNOSTIC, \ "Use parallel relocation code to speed up startup.") \ \ + /* AOT Code flags */ \ + \ + product(bool, AOTAdapterCaching, false, DIAGNOSTIC, \ + "Enable saving and restoring i2c2i adapters in AOT cache") \ + \ + product(uint, AOTCodeMaxSize, 10*M, DIAGNOSTIC, \ + "Buffer size in bytes for AOT code caching") \ + range(1*M, max_jint) \ + \ + product(bool, AbortVMOnAOTCodeFailure, false, DIAGNOSTIC, \ + "Abort VM on the first occurrence of AOT code load or store " \ + "failure. By default VM will continue execute without AOT code.") \ + \ + develop(bool, TestAOTAdapterLinkFailure, false, \ + "Test failure of adapter linking when loading from AOT cache.") \ + // end of CDS_FLAGS DECLARE_FLAGS(CDS_FLAGS) diff --git a/src/hotspot/share/cds/cppVtables.cpp b/src/hotspot/share/cds/cppVtables.cpp index b8243cedf6d..7d484c8d327 100644 --- a/src/hotspot/share/cds/cppVtables.cpp +++ b/src/hotspot/share/cds/cppVtables.cpp @@ -281,6 +281,8 @@ intptr_t* CppVtables::get_archived_vtable(MetaspaceObj::Type msotype, address ob case MetaspaceObj::AnnotationsType: case MetaspaceObj::MethodCountersType: case MetaspaceObj::RecordComponentType: + case MetaspaceObj::AdapterHandlerEntryType: + case MetaspaceObj::AdapterFingerPrintType: // These have no vtables. break; case MetaspaceObj::MethodDataType: diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp index 1b5d455485c..fbef3c9532d 100644 --- a/src/hotspot/share/cds/filemap.cpp +++ b/src/hotspot/share/cds/filemap.cpp @@ -824,7 +824,7 @@ bool FileMapRegion::check_region_crc(char* base) const { static const char* region_name(int region_index) { static const char* names[] = { - "rw", "ro", "bm", "hp" + "rw", "ro", "bm", "hp", "ac" }; const int num_regions = sizeof(names)/sizeof(names[0]); assert(0 <= region_index && region_index < num_regions, "sanity"); @@ -910,6 +910,9 @@ void FileMapInfo::write_region(int region, char* base, size_t size, " bytes, addr " INTPTR_FORMAT " file offset 0x%08" PRIxPTR " crc 0x%08x", region_name(region), region, size, p2i(requested_base), _file_offset, crc); + } else { + log_info(cds)("Shared file region (%s) %d: %8zu" + " bytes", region_name(region), region, size); } r->init(region, mapping_offset, size, read_only, allow_exec, crc); @@ -1111,7 +1114,7 @@ bool FileMapInfo::remap_shared_readonly_as_readwrite() { } // Memory map a region in the address space. -static const char* shared_region_name[] = { "ReadWrite", "ReadOnly", "Bitmap", "Heap" }; +static const char* shared_region_name[] = { "ReadWrite", "ReadOnly", "Bitmap", "Heap", "Code" }; MapArchiveResult FileMapInfo::map_regions(int regions[], int num_regions, char* mapped_base_address, ReservedSpace rs) { DEBUG_ONLY(FileMapRegion* last_region = nullptr); @@ -1274,6 +1277,42 @@ char* FileMapInfo::map_bitmap_region() { return bitmap_base; } +bool FileMapInfo::map_aot_code_region(ReservedSpace rs) { + FileMapRegion* r = region_at(MetaspaceShared::ac); + assert(r->used() > 0 && r->used_aligned() == rs.size(), "must be"); + + char* requested_base = rs.base(); + assert(requested_base != nullptr, "should be inside code cache"); + + char* mapped_base; + if (MetaspaceShared::use_windows_memory_mapping()) { + if (!read_region(MetaspaceShared::ac, requested_base, r->used_aligned(), /* do_commit = */ true)) { + log_info(cds)("Failed to read aot code shared space into reserved space at " INTPTR_FORMAT, + p2i(requested_base)); + return false; + } + mapped_base = requested_base; + } else { + // We do not execute in-place in the AOT code region. + // AOT code is copied to the CodeCache for execution. + bool read_only = false, allow_exec = false; + mapped_base = map_memory(_fd, _full_path, r->file_offset(), + requested_base, r->used_aligned(), read_only, allow_exec, mtClassShared); + } + if (mapped_base == nullptr) { + log_info(cds)("failed to map aot code region"); + return false; + } else { + assert(mapped_base == requested_base, "must be"); + r->set_mapped_from_file(true); + r->set_mapped_base(mapped_base); + log_info(cds)("Mapped static region #%d at base " INTPTR_FORMAT " top " INTPTR_FORMAT " (%s)", + MetaspaceShared::ac, p2i(r->mapped_base()), p2i(r->mapped_end()), + shared_region_name[MetaspaceShared::ac]); + return true; + } +} + class SharedDataRelocationTask : public ArchiveWorkerTask { private: BitMapView* const _rw_bm; diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp index 8793e110948..d90be1ccff3 100644 --- a/src/hotspot/share/cds/filemap.hpp +++ b/src/hotspot/share/cds/filemap.hpp @@ -368,6 +368,7 @@ public: MemRegion get_heap_region_requested_range() NOT_CDS_JAVA_HEAP_RETURN_(MemRegion()); bool read_region(int i, char* base, size_t size, bool do_commit); char* map_bitmap_region(); + bool map_aot_code_region(ReservedSpace rs); void unmap_region(int i); void close(); bool is_open() { return _file_open; } diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp index bdec6441080..5f7b496f8d2 100644 --- a/src/hotspot/share/cds/metaspaceShared.cpp +++ b/src/hotspot/share/cds/metaspaceShared.cpp @@ -59,6 +59,7 @@ #include "classfile/systemDictionaryShared.hpp" #include "classfile/vmClasses.hpp" #include "classfile/vmSymbols.hpp" +#include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" #include "gc/shared/gcVMOperations.hpp" #include "interpreter/bytecodeStream.hpp" @@ -491,6 +492,8 @@ void MetaspaceShared::serialize(SerializeClosure* soc) { soc->do_ptr((void**)&_archived_method_handle_intrinsics); LambdaFormInvokers::serialize(soc); + AdapterHandlerLibrary::serialize_shared_table_header(soc); + soc->do_tag(666); } @@ -609,6 +612,10 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables(AOTClassLocationConfig*& // Write lambform lines into archive LambdaFormInvokers::dump_static_archive_invokers(); + if (AOTCodeCache::is_dumping_adapters()) { + AdapterHandlerLibrary::dump_aot_adapter_table(); + } + // Write the other data to the output array. DumpRegion* ro_region = ArchiveBuilder::current()->ro_region(); char* start = ro_region->top(); @@ -998,6 +1005,17 @@ void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS VM_PopulateDumpSharedSpace op(builder); VMThread::execute(&op); + if (AOTCodeCache::is_on_for_dump() && CDSConfig::is_dumping_final_static_archive()) { + CDSConfig::enable_dumping_aot_code(); + { + builder.start_ac_region(); + // Write the contents to AOT code region and close AOTCodeCache before packing the region + AOTCodeCache::close(); + builder.end_ac_region(); + } + CDSConfig::disable_dumping_aot_code(); + } + if (!write_static_archive(&builder, op.map_info(), op.heap_info())) { THROW_MSG(vmSymbols::java_io_IOException(), "Encountered error while dumping"); } @@ -1769,6 +1787,7 @@ void MetaspaceShared::initialize_shared_spaces() { static_mapinfo->patch_heap_embedded_pointers(); ArchiveHeapLoader::finish_initialization(); Universe::load_archived_object_instances(); + AOTCodeCache::initialize(); // Close the mapinfo file static_mapinfo->close(); @@ -1820,6 +1839,11 @@ void MetaspaceShared::initialize_shared_spaces() { SystemDictionaryShared::print_shared_archive(tty, false/*dynamic*/); } + if (AOTCodeCache::is_on_for_use()) { + tty->print_cr("\n\nAOT Code"); + AOTCodeCache::print_on(tty); + } + // collect shared symbols and strings CountSharedSymbols cl; SymbolTable::shared_symbols_do(&cl); diff --git a/src/hotspot/share/cds/metaspaceShared.hpp b/src/hotspot/share/cds/metaspaceShared.hpp index e03994be1b9..875801cc0e6 100644 --- a/src/hotspot/share/cds/metaspaceShared.hpp +++ b/src/hotspot/share/cds/metaspaceShared.hpp @@ -67,8 +67,9 @@ class MetaspaceShared : AllStatic { ro = 1, // read-only shared space bm = 2, // relocation bitmaps (freed after file mapping is finished) hp = 3, // heap region + ac = 4, // aot code num_core_region = 2, // rw and ro - n_regions = 4 // total number of regions + n_regions = 5 // total number of regions }; static void preload_and_dump(TRAPS) NOT_CDS_RETURN; diff --git a/src/hotspot/share/classfile/compactHashtable.hpp b/src/hotspot/share/classfile/compactHashtable.hpp index 2985f0f9a1a..47fbd06f629 100644 --- a/src/hotspot/share/classfile/compactHashtable.hpp +++ b/src/hotspot/share/classfile/compactHashtable.hpp @@ -282,7 +282,15 @@ public: } template - inline void iterate(ITER* iter) const { + inline void iterate(ITER* iter) const { iterate([&](V v) { iter->do_value(v); }); } + + template + inline void iterate(const Function& function) const { // lambda enabled API + iterate(const_cast(function)); + } + + template + inline void iterate(Function& function) const { // lambda enabled API for (u4 i = 0; i < _bucket_count; i++) { u4 bucket_info = _buckets[i]; u4 bucket_offset = BUCKET_OFFSET(bucket_info); @@ -290,11 +298,11 @@ public: u4* entry = _entries + bucket_offset; if (bucket_type == VALUE_ONLY_BUCKET_TYPE) { - iter->do_value(decode(entry[0])); + function(decode(entry[0])); } else { - u4*entry_max = _entries + BUCKET_OFFSET(_buckets[i + 1]); + u4* entry_max = _entries + BUCKET_OFFSET(_buckets[i + 1]); while (entry < entry_max) { - iter->do_value(decode(entry[1])); + function(decode(entry[1])); entry += 2; } } diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp new file mode 100644 index 00000000000..e94c9ac8665 --- /dev/null +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -0,0 +1,1324 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "cds/aotCacheAccess.hpp" +#include "cds/cds_globals.hpp" +#include "cds/cdsConfig.hpp" +#include "cds/heapShared.hpp" +#include "cds/metaspaceShared.hpp" +#include "classfile/javaAssertions.hpp" +#include "code/aotCodeCache.hpp" +#include "code/codeCache.hpp" +#include "gc/shared/gcConfig.hpp" +#include "logging/logStream.hpp" +#include "memory/memoryReserver.hpp" +#include "runtime/flags/flagSetting.hpp" +#include "runtime/globals_extension.hpp" +#include "runtime/java.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/os.inline.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#ifdef COMPILER2 +#include "opto/runtime.hpp" +#endif +#if INCLUDE_G1GC +#include "gc/g1/g1BarrierSetRuntime.hpp" +#endif +#if INCLUDE_ZGC +#include "gc/z/zBarrierSetRuntime.hpp" +#endif + +#include +#include + +static void report_load_failure() { + if (AbortVMOnAOTCodeFailure) { + vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr); + } + log_info(aot, codecache, init)("Unable to use AOT Code Cache."); + AOTAdapterCaching = false; +} + +static void report_store_failure() { + if (AbortVMOnAOTCodeFailure) { + tty->print_cr("Unable to create AOT Code Cache."); + vm_abort(false); + } + log_info(aot, codecache, exit)("Unable to create AOT Code Cache."); + AOTAdapterCaching = false; +} + +bool AOTCodeCache::is_dumping_adapters() { + return AOTAdapterCaching && is_on_for_dump(); +} + +bool AOTCodeCache::is_using_adapters() { + return AOTAdapterCaching && is_on_for_use(); +} + +static uint _max_aot_code_size = 0; +uint AOTCodeCache::max_aot_code_size() { + return _max_aot_code_size; +} + +void AOTCodeCache::initialize() { + if (FLAG_IS_DEFAULT(AOTCache)) { + log_info(aot, codecache, init)("AOT Code Cache is not used: AOTCache is not specified."); + return; // AOTCache must be specified to dump and use AOT code + } + + bool is_dumping = false; + bool is_using = false; + if (CDSConfig::is_dumping_final_static_archive() && CDSConfig::is_dumping_aot_linked_classes()) { + FLAG_SET_ERGO_IF_DEFAULT(AOTAdapterCaching, true); + is_dumping = true; + } else if (CDSConfig::is_using_archive() && CDSConfig::is_using_aot_linked_classes()) { + FLAG_SET_ERGO_IF_DEFAULT(AOTAdapterCaching, true); + is_using = true; + } else { + log_info(aot, codecache, init)("AOT Code Cache is not used: AOT Class Linking is not used."); + return; // nothing to do + } + if (!AOTAdapterCaching) { + return; // AOT code caching disabled on command line + } + _max_aot_code_size = AOTCodeMaxSize; + if (!FLAG_IS_DEFAULT(AOTCodeMaxSize)) { + if (!is_aligned(AOTCodeMaxSize, os::vm_allocation_granularity())) { + _max_aot_code_size = align_up(AOTCodeMaxSize, os::vm_allocation_granularity()); + log_debug(aot,codecache,init)("Max AOT Code Cache size is aligned up to %uK", (int)(max_aot_code_size()/K)); + } + } + size_t aot_code_size = is_using ? AOTCacheAccess::get_aot_code_region_size() : 0; + if (is_using && aot_code_size == 0) { + log_info(aot, codecache, init)("AOT Code Cache is empty"); + return; + } + if (!open_cache(is_dumping, is_using)) { + if (is_using) { + report_load_failure(); + } else { + report_store_failure(); + } + return; + } + if (is_dumping) { + FLAG_SET_DEFAULT(ForceUnreachable, true); + } + FLAG_SET_DEFAULT(DelayCompilerStubsGeneration, false); +} + +void AOTCodeCache::init2() { + if (!is_on()) { + return; + } + if (!verify_vm_config()) { + close(); + report_load_failure(); + } + // initialize the table of external routines so we can save + // generated code blobs that reference them + init_extrs_table(); +} + +AOTCodeCache* AOTCodeCache::_cache = nullptr; + +bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) { + AOTCodeCache* cache = new AOTCodeCache(is_dumping, is_using); + if (cache->failed()) { + delete cache; + _cache = nullptr; + return false; + } + _cache = cache; + return true; +} + +void AOTCodeCache::close() { + if (is_on()) { + delete _cache; // Free memory + _cache = nullptr; + } +} + +#define DATA_ALIGNMENT HeapWordSize + +AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) : + _load_header(nullptr), + _load_buffer(nullptr), + _store_buffer(nullptr), + _C_store_buffer(nullptr), + _write_position(0), + _load_size(0), + _store_size(0), + _for_use(is_using), + _for_dump(is_dumping), + _closing(false), + _failed(false), + _lookup_failed(false), + _table(nullptr), + _load_entries(nullptr), + _search_entries(nullptr), + _store_entries(nullptr), + _C_strings_buf(nullptr), + _store_entries_cnt(0) +{ + // Read header at the begining of cache + if (_for_use) { + // Read cache + size_t load_size = AOTCacheAccess::get_aot_code_region_size(); + ReservedSpace rs = MemoryReserver::reserve(load_size, mtCode); + if (!rs.is_reserved()) { + log_warning(aot, codecache, init)("Failed to reserved %u bytes of memory for mapping AOT code region into AOT Code Cache", (uint)load_size); + set_failed(); + return; + } + if (!AOTCacheAccess::map_aot_code_region(rs)) { + log_warning(aot, codecache, init)("Failed to read/mmap cached code region into AOT Code Cache"); + set_failed(); + return; + } + + _load_size = (uint)load_size; + _load_buffer = (char*)rs.base(); + assert(is_aligned(_load_buffer, DATA_ALIGNMENT), "load_buffer is not aligned"); + log_debug(aot, codecache, init)("Mapped %u bytes at address " INTPTR_FORMAT " at AOT Code Cache", _load_size, p2i(_load_buffer)); + + _load_header = (Header*)addr(0); + if (!_load_header->verify_config(_load_size)) { + set_failed(); + return; + } + log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", _load_header->entries_count()); + log_debug(aot, codecache, init)(" Adapters: total=%u", _load_header->adapters_count()); + log_debug(aot, codecache, init)(" All Blobs: total=%u", _load_header->blobs_count()); + log_debug(aot, codecache, init)(" AOT code cache size: %u bytes", _load_header->cache_size()); + + // Read strings + load_strings(); + } + if (_for_dump) { + _C_store_buffer = NEW_C_HEAP_ARRAY(char, max_aot_code_size() + DATA_ALIGNMENT, mtCode); + _store_buffer = align_up(_C_store_buffer, DATA_ALIGNMENT); + // Entries allocated at the end of buffer in reverse (as on stack). + _store_entries = (AOTCodeEntry*)align_up(_C_store_buffer + max_aot_code_size(), DATA_ALIGNMENT); + log_debug(aot, codecache, init)("Allocated store buffer at address " INTPTR_FORMAT " of size %u", p2i(_store_buffer), max_aot_code_size()); + } + _table = new AOTCodeAddressTable(); +} + +void AOTCodeCache::init_extrs_table() { + AOTCodeAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_extrs(); + } +} + +void AOTCodeCache::init_shared_blobs_table() { + AOTCodeAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_shared_blobs(); + } +} + +AOTCodeCache::~AOTCodeCache() { + if (_closing) { + return; // Already closed + } + // Stop any further access to cache. + _closing = true; + + MutexLocker ml(Compile_lock); + if (for_dump()) { // Finalize cache + finish_write(); + } + _load_buffer = nullptr; + if (_C_store_buffer != nullptr) { + FREE_C_HEAP_ARRAY(char, _C_store_buffer); + _C_store_buffer = nullptr; + _store_buffer = nullptr; + } + if (_table != nullptr) { + delete _table; + _table = nullptr; + } +} + +void AOTCodeCache::Config::record() { + _flags = 0; +#ifdef ASSERT + _flags |= debugVM; +#endif + if (UseCompressedOops) { + _flags |= compressedOops; + } + if (UseCompressedClassPointers) { + _flags |= compressedClassPointers; + } + if (UseTLAB) { + _flags |= useTLAB; + } + if (JavaAssertions::systemClassDefault()) { + _flags |= systemClassAssertions; + } + if (JavaAssertions::userClassDefault()) { + _flags |= userClassAssertions; + } + if (EnableContended) { + _flags |= enableContendedPadding; + } + if (RestrictContended) { + _flags |= restrictContendedPadding; + } + _compressedOopShift = CompressedOops::shift(); + _compressedKlassShift = CompressedKlassPointers::shift(); + _contendedPaddingWidth = ContendedPaddingWidth; + _objectAlignment = ObjectAlignmentInBytes; + _gc = (uint)Universe::heap()->kind(); +} + +bool AOTCodeCache::Config::verify() const { +#ifdef ASSERT + if ((_flags & debugVM) == 0) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by product VM, it can't be used by debug VM"); + return false; + } +#else + if ((_flags & debugVM) != 0) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by debug VM, it can't be used by product VM"); + return false; + } +#endif + + CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc; + if (aot_gc != Universe::heap()->kind()) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name()); + return false; + } + + if (((_flags & compressedOops) != 0) != UseCompressedOops) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedOops = %s", UseCompressedOops ? "false" : "true"); + return false; + } + if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedClassPointers = %s", UseCompressedClassPointers ? "false" : "true"); + return false; + } + + if (((_flags & systemClassAssertions) != 0) != JavaAssertions::systemClassDefault()) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::systemClassDefault() = %s", JavaAssertions::systemClassDefault() ? "disabled" : "enabled"); + return false; + } + if (((_flags & userClassAssertions) != 0) != JavaAssertions::userClassDefault()) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with JavaAssertions::userClassDefault() = %s", JavaAssertions::userClassDefault() ? "disabled" : "enabled"); + return false; + } + + if (((_flags & enableContendedPadding) != 0) != EnableContended) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with EnableContended = %s", EnableContended ? "false" : "true"); + return false; + } + if (((_flags & restrictContendedPadding) != 0) != RestrictContended) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with RestrictContended = %s", RestrictContended ? "false" : "true"); + return false; + } + if (_compressedOopShift != (uint)CompressedOops::shift()) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedOops::shift() = %d vs current %d", _compressedOopShift, CompressedOops::shift()); + return false; + } + if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift()); + return false; + } + if (_contendedPaddingWidth != (uint)ContendedPaddingWidth) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ContendedPaddingWidth = %d vs current %d", _contendedPaddingWidth, ContendedPaddingWidth); + return false; + } + if (_objectAlignment != (uint)ObjectAlignmentInBytes) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with ObjectAlignmentInBytes = %d vs current %d", _objectAlignment, ObjectAlignmentInBytes); + return false; + } + return true; +} + +bool AOTCodeCache::Header::verify_config(uint load_size) const { + if (_version != AOT_CODE_VERSION) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version); + return false; + } + if (load_size < _cache_size) { + log_debug(aot, codecache, init)("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size); + return false; + } + return true; +} + +AOTCodeCache* AOTCodeCache::open_for_use() { + if (AOTCodeCache::is_on_for_use()) { + return AOTCodeCache::cache(); + } + return nullptr; +} + +AOTCodeCache* AOTCodeCache::open_for_dump() { + if (AOTCodeCache::is_on_for_dump()) { + AOTCodeCache* cache = AOTCodeCache::cache(); + cache->clear_lookup_failed(); // Reset bit + return cache; + } + return nullptr; +} + +void copy_bytes(const char* from, address to, uint size) { + assert(size > 0, "sanity"); + bool by_words = true; + if ((size > 2 * HeapWordSize) && (((intptr_t)from | (intptr_t)to) & (HeapWordSize - 1)) == 0) { + // Use wordwise copies if possible: + Copy::disjoint_words((HeapWord*)from, + (HeapWord*)to, + ((size_t)size + HeapWordSize-1) / HeapWordSize); + } else { + by_words = false; + Copy::conjoint_jbytes(from, to, (size_t)size); + } + log_trace(aot, codecache)("Copied %d bytes as %s from " INTPTR_FORMAT " to " INTPTR_FORMAT, size, (by_words ? "HeapWord" : "bytes"), p2i(from), p2i(to)); +} + +AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry) { + _cache = cache; + _entry = entry; + _load_buffer = cache->cache_buffer(); + _read_position = 0; + _lookup_failed = false; +} + +void AOTCodeReader::set_read_position(uint pos) { + if (pos == _read_position) { + return; + } + assert(pos < _cache->load_size(), "offset:%d >= file size:%d", pos, _cache->load_size()); + _read_position = pos; +} + +bool AOTCodeCache::set_write_position(uint pos) { + if (pos == _write_position) { + return true; + } + if (_store_size < _write_position) { + _store_size = _write_position; // Adjust during write + } + assert(pos < _store_size, "offset:%d >= file size:%d", pos, _store_size); + _write_position = pos; + return true; +} + +static char align_buffer[256] = { 0 }; + +bool AOTCodeCache::align_write() { + // We are not executing code from cache - we copy it by bytes first. + // No need for big alignment (or at all). + uint padding = DATA_ALIGNMENT - (_write_position & (DATA_ALIGNMENT - 1)); + if (padding == DATA_ALIGNMENT) { + return true; + } + uint n = write_bytes((const void*)&align_buffer, padding); + if (n != padding) { + return false; + } + log_trace(aot, codecache)("Adjust write alignment in AOT Code Cache"); + return true; +} + +// Check to see if AOT code cache has required space to store "nbytes" of data +address AOTCodeCache::reserve_bytes(uint nbytes) { + assert(for_dump(), "Code Cache file is not created"); + uint new_position = _write_position + nbytes; + if (new_position >= (uint)((char*)_store_entries - _store_buffer)) { + log_warning(aot,codecache)("Failed to ensure %d bytes at offset %d in AOT Code Cache. Increase AOTCodeMaxSize.", + nbytes, _write_position); + set_failed(); + report_store_failure(); + return nullptr; + } + address buffer = (address)(_store_buffer + _write_position); + log_trace(aot, codecache)("Reserved %d bytes at offset %d in AOT Code Cache", nbytes, _write_position); + _write_position += nbytes; + if (_store_size < _write_position) { + _store_size = _write_position; + } + return buffer; +} + +uint AOTCodeCache::write_bytes(const void* buffer, uint nbytes) { + assert(for_dump(), "Code Cache file is not created"); + if (nbytes == 0) { + return 0; + } + uint new_position = _write_position + nbytes; + if (new_position >= (uint)((char*)_store_entries - _store_buffer)) { + log_warning(aot, codecache)("Failed to write %d bytes at offset %d to AOT Code Cache. Increase AOTCodeMaxSize.", + nbytes, _write_position); + set_failed(); + report_store_failure(); + return 0; + } + copy_bytes((const char* )buffer, (address)(_store_buffer + _write_position), nbytes); + log_trace(aot, codecache)("Wrote %d bytes at offset %d to AOT Code Cache", nbytes, _write_position); + _write_position += nbytes; + if (_store_size < _write_position) { + _store_size = _write_position; + } + return nbytes; +} + +void* AOTCodeEntry::operator new(size_t x, AOTCodeCache* cache) { + return (void*)(cache->add_entry()); +} + +static bool check_entry(AOTCodeEntry::Kind kind, uint id, AOTCodeEntry* entry) { + if (entry->kind() == kind) { + assert(entry->id() == id, "sanity"); + return true; // Found + } + return false; +} + +AOTCodeEntry* AOTCodeCache::find_entry(AOTCodeEntry::Kind kind, uint id) { + assert(_for_use, "sanity"); + uint count = _load_header->entries_count(); + if (_load_entries == nullptr) { + // Read it + _search_entries = (uint*)addr(_load_header->entries_offset()); // [id, index] + _load_entries = (AOTCodeEntry*)(_search_entries + 2 * count); + log_debug(aot, codecache, init)("Read %d entries table at offset %d from AOT Code Cache", count, _load_header->entries_offset()); + } + // Binary search + int l = 0; + int h = count - 1; + while (l <= h) { + int mid = (l + h) >> 1; + int ix = mid * 2; + uint is = _search_entries[ix]; + if (is == id) { + int index = _search_entries[ix + 1]; + AOTCodeEntry* entry = &(_load_entries[index]); + if (check_entry(kind, id, entry)) { + return entry; // Found + } + break; // Not found match + } else if (is < id) { + l = mid + 1; + } else { + h = mid - 1; + } + } + return nullptr; +} + +extern "C" { + static int uint_cmp(const void *i, const void *j) { + uint a = *(uint *)i; + uint b = *(uint *)j; + return a > b ? 1 : a < b ? -1 : 0; + } +} + +bool AOTCodeCache::finish_write() { + if (!align_write()) { + return false; + } + uint strings_offset = _write_position; + int strings_count = store_strings(); + if (strings_count < 0) { + return false; + } + if (!align_write()) { + return false; + } + uint strings_size = _write_position - strings_offset; + + uint entries_count = 0; // Number of entrant (useful) code entries + uint entries_offset = _write_position; + + uint store_count = _store_entries_cnt; + if (store_count > 0) { + uint header_size = (uint)align_up(sizeof(AOTCodeCache::Header), DATA_ALIGNMENT); + uint code_count = store_count; + uint search_count = code_count * 2; + uint search_size = search_count * sizeof(uint); + uint entries_size = (uint)align_up(code_count * sizeof(AOTCodeEntry), DATA_ALIGNMENT); // In bytes + // _write_position includes size of code and strings + uint code_alignment = code_count * DATA_ALIGNMENT; // We align_up code size when storing it. + uint total_size = header_size + _write_position + code_alignment + search_size + entries_size; + assert(total_size < max_aot_code_size(), "AOT Code size (" UINT32_FORMAT " bytes) is greater than AOTCodeMaxSize(" UINT32_FORMAT " bytes).", total_size, max_aot_code_size()); + + // Create ordered search table for entries [id, index]; + uint* search = NEW_C_HEAP_ARRAY(uint, search_count, mtCode); + // Allocate in AOT Cache buffer + char* buffer = (char *)AOTCacheAccess::allocate_aot_code_region(total_size + DATA_ALIGNMENT); + char* start = align_up(buffer, DATA_ALIGNMENT); + char* current = start + header_size; // Skip header + + AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry + uint adapters_count = 0; + uint blobs_count = 0; + uint max_size = 0; + // AOTCodeEntry entries were allocated in reverse in store buffer. + // Process them in reverse order to cache first code first. + for (int i = store_count - 1; i >= 0; i--) { + entries_address[i].set_next(nullptr); // clear pointers before storing data + uint size = align_up(entries_address[i].size(), DATA_ALIGNMENT); + if (size > max_size) { + max_size = size; + } + copy_bytes((_store_buffer + entries_address[i].offset()), (address)current, size); + entries_address[i].set_offset(current - start); // New offset + current += size; + uint n = write_bytes(&(entries_address[i]), sizeof(AOTCodeEntry)); + if (n != sizeof(AOTCodeEntry)) { + FREE_C_HEAP_ARRAY(uint, search); + return false; + } + search[entries_count*2 + 0] = entries_address[i].id(); + search[entries_count*2 + 1] = entries_count; + entries_count++; + AOTCodeEntry::Kind kind = entries_address[i].kind(); + if (kind == AOTCodeEntry::Adapter) { + adapters_count++; + } else if (kind == AOTCodeEntry::Blob) { + blobs_count++; + } + } + if (entries_count == 0) { + log_info(aot, codecache, exit)("AOT Code Cache was not created: no entires"); + FREE_C_HEAP_ARRAY(uint, search); + return true; // Nothing to write + } + assert(entries_count <= store_count, "%d > %d", entries_count, store_count); + // Write strings + if (strings_count > 0) { + copy_bytes((_store_buffer + strings_offset), (address)current, strings_size); + strings_offset = (current - start); // New offset + current += strings_size; + } + + uint new_entries_offset = (current - start); // New offset + // Sort and store search table + qsort(search, entries_count, 2*sizeof(uint), uint_cmp); + search_size = 2 * entries_count * sizeof(uint); + copy_bytes((const char*)search, (address)current, search_size); + FREE_C_HEAP_ARRAY(uint, search); + current += search_size; + + // Write entries + entries_size = entries_count * sizeof(AOTCodeEntry); // New size + copy_bytes((_store_buffer + entries_offset), (address)current, entries_size); + current += entries_size; + uint size = (current - start); + assert(size <= total_size, "%d > %d", size , total_size); + + log_debug(aot, codecache, exit)(" Adapters: total=%u", adapters_count); + log_debug(aot, codecache, exit)(" All Blobs: total=%u", blobs_count); + log_debug(aot, codecache, exit)(" AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size); + + // Finalize header + AOTCodeCache::Header* header = (AOTCodeCache::Header*)start; + header->init(size, (uint)strings_count, strings_offset, + entries_count, new_entries_offset, + adapters_count, blobs_count); + + log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", entries_count); + } + return true; +} + +//------------------Store/Load AOT code ---------------------- + +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name, int entry_offset_count, int* entry_offsets) { + AOTCodeCache* cache = open_for_dump(); + if (cache == nullptr) { + return false; + } + assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + + if ((entry_kind == AOTCodeEntry::Adapter) && !AOTAdapterCaching) { + return false; + } + log_debug(aot, codecache, stubs)("Writing blob '%s' to AOT Code Cache", name); + +#ifdef ASSERT + LogStreamHandle(Trace, aot, codecache, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + blob.print_on(&log); + } +#endif + // we need to take a lock to prevent race between compiler threads generating AOT code + // and the main thread generating adapter + MutexLocker ml(Compile_lock); + if (!cache->align_write()) { + return false; + } + uint entry_position = cache->_write_position; + + // Write name + uint name_offset = cache->_write_position - entry_position; + uint name_size = (uint)strlen(name) + 1; // Includes '/0' + uint n = cache->write_bytes(name, name_size); + if (n != name_size) { + return false; + } + + // Write CodeBlob + if (!cache->align_write()) { + return false; + } + uint blob_offset = cache->_write_position - entry_position; + address archive_buffer = cache->reserve_bytes(blob.size()); + if (archive_buffer == nullptr) { + return false; + } + CodeBlob::archive_blob(&blob, archive_buffer); + + uint reloc_data_size = blob.relocation_size(); + n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size); + if (n != reloc_data_size) { + return false; + } + + bool has_oop_maps = false; + if (blob.oop_maps() != nullptr) { + if (!cache->write_oop_map_set(blob)) { + return false; + } + has_oop_maps = true; + } + + if (!cache->write_relocations(blob)) { + return false; + } + + // Write entries offsets + n = cache->write_bytes(&entry_offset_count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + for (int i = 0; i < entry_offset_count; i++) { + uint32_t off = (uint32_t)entry_offsets[i]; + n = cache->write_bytes(&off, sizeof(uint32_t)); + if (n != sizeof(uint32_t)) { + return false; + } + } + uint entry_size = cache->_write_position - entry_position; + AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, id, + entry_position, entry_size, name_offset, name_size, + blob_offset, has_oop_maps, blob.content_begin()); + log_debug(aot, codecache, stubs)("Wrote code blob '%s(id=%d)' to AOT Code Cache", name, id); + return true; +} + +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name, int entry_offset_count, int* entry_offsets) { + AOTCodeCache* cache = open_for_use(); + if (cache == nullptr) { + return nullptr; + } + assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + + if ((entry_kind == AOTCodeEntry::Adapter) && !AOTAdapterCaching) { + return nullptr; + } + log_debug(aot, codecache, stubs)("Reading blob '%s' from AOT Code Cache", name); + + AOTCodeEntry* entry = cache->find_entry(entry_kind, id); + if (entry == nullptr) { + return nullptr; + } + AOTCodeReader reader(cache, entry); + return reader.compile_code_blob(name, entry_offset_count, entry_offsets); +} + +CodeBlob* AOTCodeReader::compile_code_blob(const char* name, int entry_offset_count, int* entry_offsets) { + uint entry_position = _entry->offset(); + + // Read name + uint name_offset = entry_position + _entry->name_offset(); + uint name_size = _entry->name_size(); // Includes '/0' + const char* stored_name = addr(name_offset); + + if (strncmp(stored_name, name, (name_size - 1)) != 0) { + log_warning(aot, codecache, stubs)("Saved blob's name '%s' is different from the expected name '%s'", + stored_name, name); + ((AOTCodeCache*)_cache)->set_failed(); + report_load_failure(); + return nullptr; + } + + // Read archived code blob + uint offset = entry_position + _entry->blob_offset(); + CodeBlob* archived_blob = (CodeBlob*)addr(offset); + offset += archived_blob->size(); + + address reloc_data = (address)addr(offset); + offset += archived_blob->relocation_size(); + set_read_position(offset); + + ImmutableOopMapSet* oop_maps = nullptr; + if (_entry->has_oop_maps()) { + oop_maps = read_oop_map_set(); + } + + CodeBlob* code_blob = CodeBlob::create(archived_blob, stored_name, reloc_data, oop_maps); + if (code_blob == nullptr) { // no space left in CodeCache + return nullptr; + } + + fix_relocations(code_blob); + + // Read entries offsets + offset = read_position(); + int stored_count = *(int*)addr(offset); + assert(stored_count == entry_offset_count, "entry offset count mismatch, count in AOT code cache=%d, expected=%d", stored_count, entry_offset_count); + offset += sizeof(int); + set_read_position(offset); + for (int i = 0; i < stored_count; i++) { + uint32_t off = *(uint32_t*)addr(offset); + offset += sizeof(uint32_t); + const char* entry_name = (_entry->kind() == AOTCodeEntry::Adapter) ? AdapterHandlerEntry::entry_name(i) : ""; + log_trace(aot, codecache, stubs)("Reading adapter '%s:%s' (0x%x) offset: 0x%x from AOT Code Cache", + stored_name, entry_name, _entry->id(), off); + entry_offsets[i] = off; + } + + log_debug(aot, codecache, stubs)("Read blob '%s' from AOT Code Cache", name); +#ifdef ASSERT + LogStreamHandle(Trace, aot, codecache, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + code_blob->print_on(&log); + } +#endif + return code_blob; +} + +// ------------ process code and data -------------- + +bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { + GrowableArray reloc_data; + RelocIterator iter(&code_blob); + LogStreamHandle(Trace, aot, codecache, reloc) log; + while (iter.next()) { + int idx = reloc_data.append(0); // default value + switch (iter.type()) { + case relocInfo::none: + break; + case relocInfo::runtime_call_type: { + // Record offset of runtime destination + CallRelocation* r = (CallRelocation*)iter.reloc(); + address dest = r->destination(); + if (dest == r->addr()) { // possible call via trampoline on Aarch64 + dest = (address)-1; // do nothing in this case when loading this relocation + } + reloc_data.at_put(idx, _table->id_for_address(dest, iter, &code_blob)); + break; + } + case relocInfo::runtime_call_w_cp_type: + fatal("runtime_call_w_cp_type unimplemented"); + break; + case relocInfo::external_word_type: { + // Record offset of runtime target + address target = ((external_word_Relocation*)iter.reloc())->target(); + reloc_data.at_put(idx, _table->id_for_address(target, iter, &code_blob)); + break; + } + case relocInfo::internal_word_type: + break; + case relocInfo::section_word_type: + break; + default: + fatal("relocation %d unimplemented", (int)iter.type()); + break; + } + if (log.is_enabled()) { + iter.print_current_on(&log); + } + } + + // Write additional relocation data: uint per relocation + // Write the count first + int count = reloc_data.length(); + write_bytes(&count, sizeof(int)); + for (GrowableArrayIterator iter = reloc_data.begin(); + iter != reloc_data.end(); ++iter) { + uint value = *iter; + int n = write_bytes(&value, sizeof(uint)); + if (n != sizeof(uint)) { + return false; + } + } + return true; +} + +void AOTCodeReader::fix_relocations(CodeBlob* code_blob) { + LogStreamHandle(Trace, aot, reloc) log; + uint offset = read_position(); + int count = *(int*)addr(offset); + offset += sizeof(int); + if (log.is_enabled()) { + log.print_cr("======== extra relocations count=%d", count); + } + uint* reloc_data = (uint*)addr(offset); + offset += (count * sizeof(uint)); + set_read_position(offset); + + RelocIterator iter(code_blob); + int j = 0; + while (iter.next()) { + switch (iter.type()) { + case relocInfo::none: + break; + case relocInfo::runtime_call_type: { + address dest = _cache->address_for_id(reloc_data[j]); + if (dest != (address)-1) { + ((CallRelocation*)iter.reloc())->set_destination(dest); + } + break; + } + case relocInfo::runtime_call_w_cp_type: + fatal("runtime_call_w_cp_type unimplemented"); + break; + case relocInfo::external_word_type: { + address target = _cache->address_for_id(reloc_data[j]); + // Add external address to global table + int index = ExternalsRecorder::find_index(target); + // Update index in relocation + Relocation::add_jint(iter.data(), index); + external_word_Relocation* reloc = (external_word_Relocation*)iter.reloc(); + assert(reloc->target() == target, "sanity"); + reloc->set_value(target); // Patch address in the code + break; + } + case relocInfo::internal_word_type: { + internal_word_Relocation* r = (internal_word_Relocation*)iter.reloc(); + r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin()); + break; + } + case relocInfo::section_word_type: { + section_word_Relocation* r = (section_word_Relocation*)iter.reloc(); + r->fix_relocation_after_aot_load(aot_code_entry()->dumptime_content_start_addr(), code_blob->content_begin()); + break; + } + default: + fatal("relocation %d unimplemented", (int)iter.type()); + break; + } + if (log.is_enabled()) { + iter.print_current_on(&log); + } + j++; + } + assert(j == count, "sanity"); +} + +bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) { + ImmutableOopMapSet* oopmaps = cb.oop_maps(); + int oopmaps_size = oopmaps->nr_of_bytes(); + if (!write_bytes(&oopmaps_size, sizeof(int))) { + return false; + } + uint n = write_bytes(oopmaps, oopmaps->nr_of_bytes()); + if (n != (uint)oopmaps->nr_of_bytes()) { + return false; + } + return true; +} + +ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() { + uint offset = read_position(); + int size = *(int *)addr(offset); + offset += sizeof(int); + ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset); + offset += size; + set_read_position(offset); + return oopmaps; +} + +//======================= AOTCodeAddressTable =============== + +// address table ids for generated routines, external addresses and C +// string addresses are partitioned into positive integer ranges +// defined by the following positive base and max values +// i.e. [_extrs_base, _extrs_base + _extrs_max -1], +// [_blobs_base, _blobs_base + _blobs_max -1], +// ... +// [_c_str_base, _c_str_base + _c_str_max -1], +#define _extrs_max 10 +#define _blobs_max 10 +#define _all_max 20 + +#define _extrs_base 0 +#define _blobs_base (_extrs_base + _extrs_max) +#define _blobs_end (_blobs_base + _blobs_max) + +#if (_blobs_end > _all_max) +#error AOTCodeAddress table ranges need adjusting +#endif + +#define SET_ADDRESS(type, addr) \ + { \ + type##_addr[type##_length++] = (address) (addr); \ + assert(type##_length <= type##_max, "increase size"); \ + } + +static bool initializing_extrs = false; + +void AOTCodeAddressTable::init_extrs() { + if (_extrs_complete || initializing_extrs) return; // Done already + initializing_extrs = true; + _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode); + + _extrs_length = 0; + + // Recored addresses of VM runtime methods + SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite); + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method); + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract); + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss); +#if INCLUDE_G1GC + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_post_entry); + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry); +#endif +#if INCLUDE_ZGC + SET_ADDRESS(_extrs, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr()); +#if defined(AMD64) + SET_ADDRESS(_extrs, &ZPointerLoadShift); +#endif +#endif +#ifdef COMPILER2 + SET_ADDRESS(_extrs, OptoRuntime::handle_exception_C); +#endif +#ifndef ZERO +#if defined(AMD64) || defined(AARCH64) || defined(RISCV64) + SET_ADDRESS(_extrs, MacroAssembler::debug64); +#endif +#endif // ZERO + + _extrs_complete = true; + log_debug(aot, codecache, init)("External addresses recorded"); +} + +static bool initializing_shared_blobs = false; + +void AOTCodeAddressTable::init_shared_blobs() { + if (_complete || initializing_shared_blobs) return; // Done already + initializing_shared_blobs = true; + _blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); + + _blobs_length = 0; // for shared blobs + + // Recored addresses of generated code blobs + SET_ADDRESS(_blobs, SharedRuntime::get_handle_wrong_method_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_ic_miss_stub()); + + _shared_blobs_complete = true; + log_debug(aot, codecache, init)("Early shared blobs recorded"); + _complete = true; +} + +#undef SET_ADDRESS + +AOTCodeAddressTable::~AOTCodeAddressTable() { + if (_extrs_addr != nullptr) { + FREE_C_HEAP_ARRAY(address, _extrs_addr); + } + if (_blobs_addr != nullptr) { + FREE_C_HEAP_ARRAY(address, _blobs_addr); + } +} + +#ifdef PRODUCT +#define MAX_STR_COUNT 200 +#else +#define MAX_STR_COUNT 500 +#endif +#define _c_str_max MAX_STR_COUNT +#define _c_str_base _all_max + +static const char* _C_strings_in[MAX_STR_COUNT] = {nullptr}; // Incoming strings +static const char* _C_strings[MAX_STR_COUNT] = {nullptr}; // Our duplicates +static int _C_strings_count = 0; +static int _C_strings_s[MAX_STR_COUNT] = {0}; +static int _C_strings_id[MAX_STR_COUNT] = {0}; +static int _C_strings_used = 0; + +void AOTCodeCache::load_strings() { + uint strings_count = _load_header->strings_count(); + if (strings_count == 0) { + return; + } + uint strings_offset = _load_header->strings_offset(); + uint* string_lengths = (uint*)addr(strings_offset); + strings_offset += (strings_count * sizeof(uint)); + uint strings_size = _load_header->entries_offset() - strings_offset; + // We have to keep cached strings longer than _cache buffer + // because they are refernced from compiled code which may + // still be executed on VM exit after _cache is freed. + char* p = NEW_C_HEAP_ARRAY(char, strings_size+1, mtCode); + memcpy(p, addr(strings_offset), strings_size); + _C_strings_buf = p; + assert(strings_count <= MAX_STR_COUNT, "sanity"); + for (uint i = 0; i < strings_count; i++) { + _C_strings[i] = p; + uint len = string_lengths[i]; + _C_strings_s[i] = i; + _C_strings_id[i] = i; + p += len; + } + assert((uint)(p - _C_strings_buf) <= strings_size, "(" INTPTR_FORMAT " - " INTPTR_FORMAT ") = %d > %d ", p2i(p), p2i(_C_strings_buf), (uint)(p - _C_strings_buf), strings_size); + _C_strings_count = strings_count; + _C_strings_used = strings_count; + log_debug(aot, codecache, init)(" Loaded %d C strings of total length %d at offset %d from AOT Code Cache", _C_strings_count, strings_size, strings_offset); +} + +int AOTCodeCache::store_strings() { + if (_C_strings_used > 0) { + uint offset = _write_position; + uint length = 0; + uint* lengths = (uint *)reserve_bytes(sizeof(uint) * _C_strings_used); + if (lengths == nullptr) { + return -1; + } + for (int i = 0; i < _C_strings_used; i++) { + const char* str = _C_strings[_C_strings_s[i]]; + uint len = (uint)strlen(str) + 1; + length += len; + assert(len < 1000, "big string: %s", str); + lengths[i] = len; + uint n = write_bytes(str, len); + if (n != len) { + return -1; + } + } + log_debug(aot, codecache, exit)(" Wrote %d C strings of total length %d at offset %d to AOT Code Cache", + _C_strings_used, length, offset); + } + return _C_strings_used; +} + +const char* AOTCodeCache::add_C_string(const char* str) { + if (is_on_for_dump() && str != nullptr) { + return _cache->_table->add_C_string(str); + } + return str; +} + +const char* AOTCodeAddressTable::add_C_string(const char* str) { + if (_extrs_complete) { + LogStreamHandle(Trace, aot, codecache, stringtable) log; // ctor outside lock + MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag); + // Check previous strings address + for (int i = 0; i < _C_strings_count; i++) { + if (_C_strings_in[i] == str) { + return _C_strings[i]; // Found previous one - return our duplicate + } else if (strcmp(_C_strings[i], str) == 0) { + return _C_strings[i]; + } + } + // Add new one + if (_C_strings_count < MAX_STR_COUNT) { + // Passed in string can be freed and used space become inaccessible. + // Keep original address but duplicate string for future compare. + _C_strings_id[_C_strings_count] = -1; // Init + _C_strings_in[_C_strings_count] = str; + const char* dup = os::strdup(str); + _C_strings[_C_strings_count++] = dup; + if (log.is_enabled()) { + log.print_cr("add_C_string: [%d] " INTPTR_FORMAT " '%s'", _C_strings_count, p2i(dup), dup); + } + return dup; + } else { + fatal("Number of C strings >= MAX_STR_COUNT"); + } + } + return str; +} + +int AOTCodeAddressTable::id_for_C_string(address str) { + if (str == nullptr) { + return -1; + } + MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag); + for (int i = 0; i < _C_strings_count; i++) { + if (_C_strings[i] == (const char*)str) { // found + int id = _C_strings_id[i]; + if (id >= 0) { + assert(id < _C_strings_used, "%d >= %d", id , _C_strings_used); + return id; // Found recorded + } + // Not found in recorded, add new + id = _C_strings_used++; + _C_strings_s[id] = i; + _C_strings_id[i] = id; + return id; + } + } + return -1; +} + +address AOTCodeAddressTable::address_for_C_string(int idx) { + assert(idx < _C_strings_count, "sanity"); + return (address)_C_strings[idx]; +} + +static int search_address(address addr, address* table, uint length) { + for (int i = 0; i < (int)length; i++) { + if (table[i] == addr) { + return i; + } + } + return -1; +} + +address AOTCodeAddressTable::address_for_id(int idx) { + if (!_extrs_complete) { + fatal("AOT Code Cache VM runtime addresses table is not complete"); + } + if (idx == -1) { + return (address)-1; + } + uint id = (uint)idx; + // special case for symbols based relative to os::init + if (id > (_c_str_base + _c_str_max)) { + return (address)os::init + idx; + } + if (idx < 0) { + fatal("Incorrect id %d for AOT Code Cache addresses table", id); + } + // no need to compare unsigned id against 0 + if (/* id >= _extrs_base && */ id < _extrs_length) { + return _extrs_addr[id - _extrs_base]; + } + if (id >= _blobs_base && id < _blobs_base + _blobs_length) { + return _blobs_addr[id - _blobs_base]; + } + if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) { + return address_for_C_string(id - _c_str_base); + } + fatal("Incorrect id %d for AOT Code Cache addresses table", id); + return nullptr; +} + +int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* code_blob) { + if (!_extrs_complete) { + fatal("AOT Code Cache VM runtime addresses table is not complete"); + } + int id = -1; + if (addr == (address)-1) { // Static call stub has jump to itself + return id; + } + // Seach for C string + id = id_for_C_string(addr); + if (id >= 0) { + return id + _c_str_base; + } + if (StubRoutines::contains(addr)) { + // Search in stubs + StubCodeDesc* desc = StubCodeDesc::desc_for(addr); + if (desc == nullptr) { + desc = StubCodeDesc::desc_for(addr + frame::pc_return_offset); + } + const char* sub_name = (desc != nullptr) ? desc->name() : ""; + fatal("Address " INTPTR_FORMAT " for Stub:%s is missing in AOT Code Cache addresses table", p2i(addr), sub_name); + } else { + CodeBlob* cb = CodeCache::find_blob(addr); + if (cb != nullptr) { + // Search in code blobs + int id_base = _blobs_base; + id = search_address(addr, _blobs_addr, _blobs_length); + if (id < 0) { + fatal("Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name()); + } else { + return id_base + id; + } + } else { + // Search in runtime functions + id = search_address(addr, _extrs_addr, _extrs_length); + if (id < 0) { + ResourceMark rm; + const int buflen = 1024; + char* func_name = NEW_RESOURCE_ARRAY(char, buflen); + int offset = 0; + if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) { + if (offset > 0) { + // Could be address of C string + uint dist = (uint)pointer_delta(addr, (address)os::init, 1); + log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table", + p2i(addr), dist, (const char*)addr); + assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance"); + return dist; + } + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + fatal("Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset); + } else { + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + os::find(addr, tty); + fatal("Address " INTPTR_FORMAT " for /('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr); + } + } else { + return _extrs_base + id; + } + } + } + return id; +} + +void AOTCodeCache::print_on(outputStream* st) { + AOTCodeCache* cache = open_for_use(); + if (cache != nullptr) { + uint count = cache->_load_header->entries_count(); + uint* search_entries = (uint*)cache->addr(cache->_load_header->entries_offset()); // [id, index] + AOTCodeEntry* load_entries = (AOTCodeEntry*)(search_entries + 2 * count); + + for (uint i = 0; i < count; i++) { + // Use search_entries[] to order ouput + int index = search_entries[2*i + 1]; + AOTCodeEntry* entry = &(load_entries[index]); + + uint entry_position = entry->offset(); + uint name_offset = entry->name_offset() + entry_position; + const char* saved_name = cache->addr(name_offset); + + st->print_cr("%4u: entry_idx:%4u Kind:%u Id:%u size=%u '%s'", + i, index, entry->kind(), entry->id(), entry->size(), saved_name); + } + } else { + st->print_cr("failed to map code cache"); + } +} + diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp new file mode 100644 index 00000000000..d5c938e3b8b --- /dev/null +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -0,0 +1,365 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_CODE_AOTCODECACHE_HPP +#define SHARE_CODE_AOTCODECACHE_HPP + +/* + * AOT Code Cache collects code from Code Cache and corresponding metadata + * during application training run. + * In following "production" runs this code and data can be loaded into + * Code Cache skipping its generation. + */ + +class CodeBuffer; +class RelocIterator; +class AOTCodeCache; +class AdapterBlob; +class ExceptionBlob; +class ImmutableOopMapSet; + +enum class vmIntrinsicID : int; +enum CompLevel : signed char; + +// Descriptor of AOT Code Cache's entry +class AOTCodeEntry { +public: + enum Kind { + None = 0, + Adapter = 1, + Blob = 2 + }; + +private: + AOTCodeEntry* _next; + Kind _kind; + uint _id; // Adapter's id, vmIntrinsic::ID for stub or name's hash for nmethod + uint _offset; // Offset to entry + uint _size; // Entry size + uint _name_offset; // Code blob name + uint _name_size; + uint _blob_offset; // Start of code in cache + bool _has_oop_maps; + address _dumptime_content_start_addr; // CodeBlob::content_begin() at dump time; used for applying relocations + +public: + AOTCodeEntry(Kind kind, uint id, + uint offset, uint size, + uint name_offset, uint name_size, + uint blob_offset, bool has_oop_maps, + address dumptime_content_start_addr) { + _next = nullptr; + _kind = kind; + _id = id; + _offset = offset; + _size = size; + _name_offset = name_offset; + _name_size = name_size; + _blob_offset = blob_offset; + _has_oop_maps = has_oop_maps; + _dumptime_content_start_addr = dumptime_content_start_addr; + } + void* operator new(size_t x, AOTCodeCache* cache); + // Delete is a NOP + void operator delete( void *ptr ) {} + + AOTCodeEntry* next() const { return _next; } + void set_next(AOTCodeEntry* next) { _next = next; } + + Kind kind() const { return _kind; } + uint id() const { return _id; } + + uint offset() const { return _offset; } + void set_offset(uint off) { _offset = off; } + + uint size() const { return _size; } + uint name_offset() const { return _name_offset; } + uint name_size() const { return _name_size; } + uint blob_offset() const { return _blob_offset; } + bool has_oop_maps() const { return _has_oop_maps; } + address dumptime_content_start_addr() const { return _dumptime_content_start_addr; } + + static bool is_valid_entry_kind(Kind kind) { return kind == Adapter || kind == Blob; } +}; + +// Addresses of stubs, blobs and runtime finctions called from compiled code. +class AOTCodeAddressTable : public CHeapObj { +private: + address* _extrs_addr; + address* _blobs_addr; + uint _extrs_length; + uint _blobs_length; + + bool _extrs_complete; + bool _shared_blobs_complete; + bool _complete; + +public: + AOTCodeAddressTable() : + _extrs_addr(nullptr), + _blobs_addr(nullptr), + _extrs_length(0), + _blobs_length(0), + _extrs_complete(false), + _shared_blobs_complete(false), + _complete(false) + { } + ~AOTCodeAddressTable(); + void init_extrs(); + void init_shared_blobs(); + const char* add_C_string(const char* str); + int id_for_C_string(address str); + address address_for_C_string(int idx); + int id_for_address(address addr, RelocIterator iter, CodeBlob* code_blob); + address address_for_id(int id); +}; + +class AOTCodeCache : public CHeapObj { + +// Classes used to describe AOT code cache. +protected: + class Config { + uint _compressedOopShift; + uint _compressedKlassShift; + uint _contendedPaddingWidth; + uint _objectAlignment; + uint _gc; + enum Flags { + none = 0, + debugVM = 1, + compressedOops = 2, + compressedClassPointers = 4, + useTLAB = 8, + systemClassAssertions = 16, + userClassAssertions = 32, + enableContendedPadding = 64, + restrictContendedPadding = 128 + }; + uint _flags; + + public: + void record(); + bool verify() const; + }; + + class Header : public CHeapObj { + private: + enum { + AOT_CODE_VERSION = 1 + }; + uint _version; // AOT code version (should match when reading code cache) + uint _cache_size; // cache size in bytes + uint _strings_count; // number of recorded C strings + uint _strings_offset; // offset to recorded C strings + uint _entries_count; // number of recorded entries + uint _entries_offset; // offset of AOTCodeEntry array describing entries + uint _adapters_count; + uint _blobs_count; + Config _config; + +public: + void init(uint cache_size, + uint strings_count, uint strings_offset, + uint entries_count, uint entries_offset, + uint adapters_count, uint blobs_count) { + _version = AOT_CODE_VERSION; + _cache_size = cache_size; + _strings_count = strings_count; + _strings_offset = strings_offset; + _entries_count = entries_count; + _entries_offset = entries_offset; + _adapters_count = adapters_count; + _blobs_count = blobs_count; + + _config.record(); + } + + + uint cache_size() const { return _cache_size; } + uint strings_count() const { return _strings_count; } + uint strings_offset() const { return _strings_offset; } + uint entries_count() const { return _entries_count; } + uint entries_offset() const { return _entries_offset; } + uint adapters_count() const { return _adapters_count; } + uint blobs_count() const { return _blobs_count; } + + bool verify_config(uint load_size) const; + bool verify_vm_config() const { // Called after Universe initialized + return _config.verify(); + } + }; + +// Continue with AOTCodeCache class definition. +private: + Header* _load_header; + char* _load_buffer; // Aligned buffer for loading cached code + char* _store_buffer; // Aligned buffer for storing cached code + char* _C_store_buffer; // Original unaligned buffer + + uint _write_position; // Position in _store_buffer + uint _load_size; // Used when reading cache + uint _store_size; // Used when writing cache + bool _for_use; // AOT cache is open for using AOT code + bool _for_dump; // AOT cache is open for dumping AOT code + bool _closing; // Closing cache file + bool _failed; // Failed read/write to/from cache (cache is broken?) + bool _lookup_failed; // Failed to lookup for info (skip only this code load) + + AOTCodeAddressTable* _table; + + AOTCodeEntry* _load_entries; // Used when reading cache + uint* _search_entries; // sorted by ID table [id, index] + AOTCodeEntry* _store_entries; // Used when writing cache + const char* _C_strings_buf; // Loaded buffer for _C_strings[] table + uint _store_entries_cnt; + + static AOTCodeCache* open_for_use(); + static AOTCodeCache* open_for_dump(); + + bool set_write_position(uint pos); + bool align_write(); + address reserve_bytes(uint nbytes); + uint write_bytes(const void* buffer, uint nbytes); + const char* addr(uint offset) const { return _load_buffer + offset; } + static AOTCodeAddressTable* addr_table() { + return is_on() && (cache()->_table != nullptr) ? cache()->_table : nullptr; + } + + void set_lookup_failed() { _lookup_failed = true; } + void clear_lookup_failed() { _lookup_failed = false; } + bool lookup_failed() const { return _lookup_failed; } + +public: + AOTCodeCache(bool is_dumping, bool is_using); + ~AOTCodeCache(); + + const char* cache_buffer() const { return _load_buffer; } + bool failed() const { return _failed; } + void set_failed() { _failed = true; } + + static uint max_aot_code_size(); + + uint load_size() const { return _load_size; } + uint write_position() const { return _write_position; } + + void load_strings(); + int store_strings(); + + static void init_extrs_table() NOT_CDS_RETURN; + static void init_shared_blobs_table() NOT_CDS_RETURN; + + address address_for_id(int id) const { return _table->address_for_id(id); } + + bool for_use() const { return _for_use && !_failed; } + bool for_dump() const { return _for_dump && !_failed; } + + bool closing() const { return _closing; } + + AOTCodeEntry* add_entry() { + _store_entries_cnt++; + _store_entries -= 1; + return _store_entries; + } + + AOTCodeEntry* find_entry(AOTCodeEntry::Kind kind, uint id); + + bool finish_write(); + + bool write_relocations(CodeBlob& code_blob); + bool write_oop_map_set(CodeBlob& cb); + + static bool store_code_blob(CodeBlob& blob, + AOTCodeEntry::Kind entry_kind, + uint id, const char* name, + int entry_offset_count, + int* entry_offsets) NOT_CDS_RETURN_(false); + + static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, + uint id, const char* name, + int entry_offset_count, + int* entry_offsets) NOT_CDS_RETURN_(nullptr); + + static uint store_entries_cnt() { + if (is_on_for_dump()) { + return cache()->_store_entries_cnt; + } + return -1; + } + +// Static access + +private: + static AOTCodeCache* _cache; + + static bool open_cache(bool is_dumping, bool is_using); + static bool verify_vm_config() { + if (is_on_for_use()) { + return _cache->_load_header->verify_vm_config(); + } + return true; + } +public: + static AOTCodeCache* cache() { return _cache; } + static void initialize() NOT_CDS_RETURN; + static void init2() NOT_CDS_RETURN; + static void close() NOT_CDS_RETURN; + static bool is_on() CDS_ONLY({ return _cache != nullptr && !_cache->closing(); }) NOT_CDS_RETURN_(false); + static bool is_on_for_use() { return is_on() && _cache->for_use(); } + static bool is_on_for_dump() { return is_on() && _cache->for_dump(); } + + static bool is_dumping_adapters() NOT_CDS_RETURN_(false); + static bool is_using_adapters() NOT_CDS_RETURN_(false); + + static const char* add_C_string(const char* str) NOT_CDS_RETURN_(str); + + static void print_on(outputStream* st) NOT_CDS_RETURN; +}; + +// Concurent AOT code reader +class AOTCodeReader { +private: + const AOTCodeCache* _cache; + const AOTCodeEntry* _entry; + const char* _load_buffer; // Loaded cached code buffer + uint _read_position; // Position in _load_buffer + uint read_position() const { return _read_position; } + void set_read_position(uint pos); + const char* addr(uint offset) const { return _load_buffer + offset; } + + bool _lookup_failed; // Failed to lookup for info (skip only this code load) + void set_lookup_failed() { _lookup_failed = true; } + void clear_lookup_failed() { _lookup_failed = false; } + bool lookup_failed() const { return _lookup_failed; } + + AOTCodeEntry* aot_code_entry() { return (AOTCodeEntry*)_entry; } +public: + AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry); + + CodeBlob* compile_code_blob(const char* name, int entry_offset_count, int* entry_offsets); + + ImmutableOopMapSet* read_oop_map_set(); + + void fix_relocations(CodeBlob* code_blob); +}; +#endif // SHARE_CODE_AOTCODECACH_HPP diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp index 4ae4ccbc858..1512f498b37 100644 --- a/src/hotspot/share/code/codeBlob.cpp +++ b/src/hotspot/share/code/codeBlob.cpp @@ -78,9 +78,12 @@ const BufferBlob::Vptr BufferBlob::_vpntr; const RuntimeStub::Vptr RuntimeStub::_vpntr; const SingletonBlob::Vptr SingletonBlob::_vpntr; const DeoptimizationBlob::Vptr DeoptimizationBlob::_vpntr; +#ifdef COMPILER2 +const ExceptionBlob::Vptr ExceptionBlob::_vpntr; +#endif // COMPILER2 const UpcallStub::Vptr UpcallStub::_vpntr; -const CodeBlob::Vptr* CodeBlob::vptr() const { +const CodeBlob::Vptr* CodeBlob::vptr(CodeBlobKind kind) { constexpr const CodeBlob::Vptr* array[(size_t)CodeBlobKind::Number_Of_Kinds] = { nullptr/* None */, &nmethod::_vpntr, @@ -98,7 +101,11 @@ const CodeBlob::Vptr* CodeBlob::vptr() const { &UpcallStub::_vpntr }; - return array[(size_t)_kind]; + return array[(size_t)kind]; +} + +const CodeBlob::Vptr* CodeBlob::vptr() const { + return vptr(_kind); } unsigned int CodeBlob::align_code_offset(int offset) { @@ -181,6 +188,19 @@ CodeBlob::CodeBlob(const char* name, CodeBlobKind kind, int size, uint16_t heade assert(_mutable_data = blob_end(), "sanity"); } +void CodeBlob::restore_mutable_data(address reloc_data) { + // Relocation data is now stored as part of the mutable data area; allocate it before copy relocations + if (_mutable_data_size > 0) { + _mutable_data = (address)os::malloc(_mutable_data_size, mtCode); + if (_mutable_data == nullptr) { + vm_exit_out_of_memory(_mutable_data_size, OOM_MALLOC_ERROR, "codebuffer: no space for mutable data"); + } + } + if (_relocation_size > 0) { + memcpy((address)relocation_begin(), reloc_data, relocation_size()); + } +} + void CodeBlob::purge() { assert(_mutable_data != nullptr, "should never be null"); if (_mutable_data != blob_end()) { @@ -215,6 +235,68 @@ void CodeBlob::print_code_on(outputStream* st) { Disassembler::decode(this, st); } +void CodeBlob::prepare_for_archiving_impl() { + set_name(nullptr); + _oop_maps = nullptr; + _mutable_data = nullptr; +#ifndef PRODUCT + asm_remarks().clear(); + dbg_strings().clear(); +#endif /* PRODUCT */ +} + +void CodeBlob::prepare_for_archiving() { + vptr(_kind)->prepare_for_archiving(this); +} + +void CodeBlob::archive_blob(CodeBlob* blob, address archive_buffer) { + blob->copy_to(archive_buffer); + CodeBlob* archived_blob = (CodeBlob*)archive_buffer; + archived_blob->prepare_for_archiving(); +} + +void CodeBlob::post_restore_impl() { + // Track memory usage statistic after releasing CodeCache_lock + MemoryService::track_code_cache_memory_usage(); +} + +void CodeBlob::post_restore() { + vptr(_kind)->post_restore(this); +} + +CodeBlob* CodeBlob::restore(address code_cache_buffer, const char* name, address archived_reloc_data, ImmutableOopMapSet* archived_oop_maps) { + copy_to(code_cache_buffer); + CodeBlob* code_blob = (CodeBlob*)code_cache_buffer; + code_blob->set_name(name); + code_blob->restore_mutable_data(archived_reloc_data); + code_blob->set_oop_maps(archived_oop_maps); + return code_blob; +} + +CodeBlob* CodeBlob::create(CodeBlob* archived_blob, const char* name, address archived_reloc_data, ImmutableOopMapSet* archived_oop_maps) { + ThreadInVMfromUnknown __tiv; // get to VM state in case we block on CodeCache_lock + + CodeCache::gc_on_allocation(); + + CodeBlob* blob = nullptr; + unsigned int size = archived_blob->size(); + { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + address code_cache_buffer = (address)CodeCache::allocate(size, CodeBlobType::NonNMethod); + if (code_cache_buffer != nullptr) { + blob = archived_blob->restore(code_cache_buffer, name, archived_reloc_data, archived_oop_maps); + assert(blob != nullptr, "sanity check"); + // Flush the code block + ICache::invalidate_range(blob->code_begin(), blob->code_size()); + CodeCache::commit(blob); // Count adapters + } + } + if (blob != nullptr) { + blob->post_restore(); + } + return blob; +} + //----------------------------------------------------------------------------------------- // Creates a RuntimeBlob from a CodeBuffer and copy code and relocation info. @@ -720,7 +802,7 @@ void CodeBlob::print_value_on(outputStream* st) const { } void CodeBlob::print_on_impl(outputStream* st) const { - st->print_cr("[CodeBlob (" INTPTR_FORMAT ")]", p2i(this)); + st->print_cr("[CodeBlob kind:%d (" INTPTR_FORMAT ")]", (int)_kind, p2i(this)); st->print_cr("Framesize: %d", _frame_size); } diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp index 1daa35c16b5..37f83824987 100644 --- a/src/hotspot/share/code/codeBlob.hpp +++ b/src/hotspot/share/code/codeBlob.hpp @@ -97,11 +97,16 @@ enum class CodeBlobKind : u1 { class UpcallStub; // for as_upcall_stub() class RuntimeStub; // for as_runtime_stub() class JavaFrameAnchor; // for UpcallStub::jfa_for_frame +class AdapterBlob; +class ExceptionBlob; class CodeBlob { friend class VMStructs; friend class JVMCIVMStructs; +private: + void restore_mutable_data(address reloc_data); + protected: // order fields from large to small to minimize padding between fields ImmutableOopMapSet* _oop_maps; // OopMap for this CodeBlob @@ -140,8 +145,15 @@ protected: public: virtual void print_on(const CodeBlob* instance, outputStream* st) const = 0; virtual void print_value_on(const CodeBlob* instance, outputStream* st) const = 0; + virtual void prepare_for_archiving(CodeBlob* instance) const { + instance->prepare_for_archiving_impl(); + }; + virtual void post_restore(CodeBlob* instance) const { + instance->post_restore_impl(); + }; }; + static const Vptr* vptr(CodeBlobKind kind); const Vptr* vptr() const; CodeBlob(const char* name, CodeBlobKind kind, CodeBuffer* cb, int size, uint16_t header_size, @@ -151,8 +163,12 @@ protected: // Simple CodeBlob used for simple BufferBlob. CodeBlob(const char* name, CodeBlobKind kind, int size, uint16_t header_size); + void operator delete(void* p) { } + void prepare_for_archiving_impl(); + void post_restore_impl(); + public: ~CodeBlob() { @@ -188,6 +204,7 @@ public: nmethod* as_nmethod_or_null() const { return is_nmethod() ? (nmethod*) this : nullptr; } nmethod* as_nmethod() const { assert(is_nmethod(), "must be nmethod"); return (nmethod*) this; } CodeBlob* as_codeblob() const { return (CodeBlob*) this; } + AdapterBlob* as_adapter_blob() const { assert(is_adapter_blob(), "must be adapter blob"); return (AdapterBlob*) this; } UpcallStub* as_upcall_stub() const { assert(is_upcall_stub(), "must be upcall stub"); return (UpcallStub*) this; } RuntimeStub* as_runtime_stub() const { assert(is_runtime_stub(), "must be runtime blob"); return (RuntimeStub*) this; } @@ -244,6 +261,7 @@ public: // OopMap for frame ImmutableOopMapSet* oop_maps() const { return _oop_maps; } void set_oop_maps(OopMapSet* p); + void set_oop_maps(ImmutableOopMapSet* p) { _oop_maps = p; } const ImmutableOopMap* oop_map_for_slot(int slot, address return_address) const; const ImmutableOopMap* oop_map_for_return_address(address return_address) const; @@ -278,6 +296,19 @@ public: void use_remarks(AsmRemarks &remarks) { _asm_remarks.share(remarks); } void use_strings(DbgStrings &strings) { _dbg_strings.share(strings); } #endif + + void copy_to(address buffer) { + memcpy(buffer, this, this->size()); + } + + // methods to archive a blob into AOT code cache + void prepare_for_archiving(); + static void archive_blob(CodeBlob* blob, address archive_buffer); + + // methods to restore a blob from AOT code cache into the CodeCache + void post_restore(); + CodeBlob* restore(address code_cache_buffer, const char* name, address archived_reloc_data, ImmutableOopMapSet* archived_oop_maps); + static CodeBlob* create(CodeBlob* archived_blob, const char* name, address archived_reloc_data, ImmutableOopMapSet* archived_oop_maps); }; //---------------------------------------------------------------------------------------------------- @@ -617,6 +648,18 @@ class ExceptionBlob: public SingletonBlob { OopMapSet* oop_maps, int frame_size ); + + void post_restore_impl() { + trace_new_stub(this, "ExceptionBlob"); + } + + class Vptr : public SingletonBlob::Vptr { + void post_restore(CodeBlob* instance) const override { + ((ExceptionBlob*)instance)->post_restore_impl(); + } + }; + + static const Vptr _vpntr; }; #endif // COMPILER2 diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 4576a348a4f..e3bd3131b45 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -3243,7 +3243,7 @@ void nmethod::print_relocations() { ResourceMark m; // in case methods get printed via the debugger tty->print_cr("relocations:"); RelocIterator iter(this); - iter.print(); + iter.print_on(tty); } #endif diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp index a828a8356aa..81fa83dbe8d 100644 --- a/src/hotspot/share/code/relocInfo.cpp +++ b/src/hotspot/share/code/relocInfo.cpp @@ -179,8 +179,31 @@ RelocIterator::RelocIterator(CodeSection* cs, address begin, address limit) { set_limits(begin, limit); } +RelocIterator::RelocIterator(CodeBlob* cb) { + initialize_misc(); + if (cb->is_nmethod()) { + _code = cb->as_nmethod(); + } else { + _code = nullptr; + } + _current = cb->relocation_begin() - 1; + _end = cb->relocation_end(); + _addr = cb->content_begin(); + + _section_start[CodeBuffer::SECT_CONSTS] = cb->content_begin(); + _section_start[CodeBuffer::SECT_INSTS] = cb->code_begin(); + + _section_end[CodeBuffer::SECT_CONSTS] = cb->code_begin(); + _section_start[CodeBuffer::SECT_INSTS] = cb->code_end(); + assert(!has_current(), "just checking"); + set_limits(nullptr, nullptr); +} + bool RelocIterator::addr_in_const() const { const int n = CodeBuffer::SECT_CONSTS; + if (_section_start[n] == nullptr) { + return false; + } return section_start(n) <= addr() && addr() < section_end(n); } @@ -471,7 +494,9 @@ void trampoline_stub_Relocation::unpack_data() { void external_word_Relocation::pack_data_to(CodeSection* dest) { short* p = (short*) dest->locs_end(); int index = ExternalsRecorder::find_index(_target); - p = pack_1_int_to(p, index); + // Use 4 bytes to store index to be able patch it when + // updating relocations in AOTCodeReader::read_relocations(). + p = add_jint(p, index); dest->set_locs_end((relocInfo*) p); } @@ -766,6 +791,14 @@ void internal_word_Relocation::fix_relocation_after_move(const CodeBuffer* src, set_value(target); } +void internal_word_Relocation::fix_relocation_after_aot_load(address orig_base_addr, address current_base_addr) { + address target = _target; + if (target == nullptr) { + target = this->target(); + target = current_base_addr + (target - orig_base_addr); + } + set_value(target); +} address internal_word_Relocation::target() { address target = _target; @@ -779,12 +812,7 @@ address internal_word_Relocation::target() { return target; } -//--------------------------------------------------------------------------------- -// Non-product code - -#ifndef PRODUCT - -static const char* reloc_type_string(relocInfo::relocType t) { +const char* relocInfo::type_name(relocInfo::relocType t) { switch (t) { #define EACH_CASE(name) \ case relocInfo::name##_type: \ @@ -802,26 +830,25 @@ static const char* reloc_type_string(relocInfo::relocType t) { } } - -void RelocIterator::print_current() { +void RelocIterator::print_current_on(outputStream* st) { if (!has_current()) { - tty->print_cr("(no relocs)"); + st->print_cr("(no relocs)"); return; } - tty->print("relocInfo@" INTPTR_FORMAT " [type=%d(%s) addr=" INTPTR_FORMAT " offset=%d", - p2i(_current), type(), reloc_type_string((relocInfo::relocType) type()), p2i(_addr), _current->addr_offset()); + st->print("relocInfo@" INTPTR_FORMAT " [type=%d(%s) addr=" INTPTR_FORMAT " offset=%d", + p2i(_current), type(), relocInfo::type_name((relocInfo::relocType) type()), p2i(_addr), _current->addr_offset()); if (current()->format() != 0) - tty->print(" format=%d", current()->format()); + st->print(" format=%d", current()->format()); if (datalen() == 1) { - tty->print(" data=%d", data()[0]); + st->print(" data=%d", data()[0]); } else if (datalen() > 0) { - tty->print(" data={"); + st->print(" data={"); for (int i = 0; i < datalen(); i++) { - tty->print("%04x", data()[i] & 0xFFFF); + st->print("%04x", data()[i] & 0xFFFF); } - tty->print("}"); + st->print("}"); } - tty->print("]"); + st->print("]"); switch (type()) { case relocInfo::oop_type: { @@ -834,14 +861,14 @@ void RelocIterator::print_current() { raw_oop = *oop_addr; oop_value = r->oop_value(); } - tty->print(" | [oop_addr=" INTPTR_FORMAT " *=" INTPTR_FORMAT "]", + st->print(" | [oop_addr=" INTPTR_FORMAT " *=" INTPTR_FORMAT "]", p2i(oop_addr), p2i(raw_oop)); // Do not print the oop by default--we want this routine to // work even during GC or other inconvenient times. if (WizardMode && oop_value != nullptr) { - tty->print("oop_value=" INTPTR_FORMAT ": ", p2i(oop_value)); + st->print("oop_value=" INTPTR_FORMAT ": ", p2i(oop_value)); if (oopDesc::is_oop(oop_value)) { - oop_value->print_value_on(tty); + oop_value->print_value_on(st); } } break; @@ -857,11 +884,11 @@ void RelocIterator::print_current() { raw_metadata = *metadata_addr; metadata_value = r->metadata_value(); } - tty->print(" | [metadata_addr=" INTPTR_FORMAT " *=" INTPTR_FORMAT "]", + st->print(" | [metadata_addr=" INTPTR_FORMAT " *=" INTPTR_FORMAT "]", p2i(metadata_addr), p2i(raw_metadata)); if (metadata_value != nullptr) { - tty->print("metadata_value=" INTPTR_FORMAT ": ", p2i(metadata_value)); - metadata_value->print_value_on(tty); + st->print("metadata_value=" INTPTR_FORMAT ": ", p2i(metadata_value)); + metadata_value->print_value_on(st); } break; } @@ -870,17 +897,17 @@ void RelocIterator::print_current() { case relocInfo::section_word_type: { DataRelocation* r = (DataRelocation*) reloc(); - tty->print(" | [target=" INTPTR_FORMAT "]", p2i(r->value())); //value==target + st->print(" | [target=" INTPTR_FORMAT "]", p2i(r->value())); //value==target break; } case relocInfo::static_call_type: { static_call_Relocation* r = (static_call_Relocation*) reloc(); - tty->print(" | [destination=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]", + st->print(" | [destination=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]", p2i(r->destination()), p2i(r->method_value())); CodeBlob* cb = CodeCache::find_blob(r->destination()); if (cb != nullptr) { - tty->print(" Blob::%s", cb->name()); + st->print(" Blob::%s", cb->name()); } break; } @@ -889,28 +916,28 @@ void RelocIterator::print_current() { { CallRelocation* r = (CallRelocation*) reloc(); address dest = r->destination(); - tty->print(" | [destination=" INTPTR_FORMAT "]", p2i(dest)); + st->print(" | [destination=" INTPTR_FORMAT "]", p2i(dest)); if (StubRoutines::contains(dest)) { StubCodeDesc* desc = StubCodeDesc::desc_for(dest); if (desc == nullptr) { desc = StubCodeDesc::desc_for(dest + frame::pc_return_offset); } if (desc != nullptr) { - tty->print(" Stub::%s", desc->name()); + st->print(" Stub::%s", desc->name()); } } else { CodeBlob* cb = CodeCache::find_blob(dest); if (cb != nullptr) { - tty->print(" %s", cb->name()); + st->print(" %s", cb->name()); } else { ResourceMark rm; const int buflen = 1024; char* buf = NEW_RESOURCE_ARRAY(char, buflen); int offset; if (os::dll_address_to_function_name(dest, buf, buflen, &offset)) { - tty->print(" %s", buf); + st->print(" %s", buf); if (offset != 0) { - tty->print("+%d", offset); + st->print("+%d", offset); } } } @@ -920,45 +947,45 @@ void RelocIterator::print_current() { case relocInfo::virtual_call_type: { virtual_call_Relocation* r = (virtual_call_Relocation*) reloc(); - tty->print(" | [destination=" INTPTR_FORMAT " cached_value=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]", + st->print(" | [destination=" INTPTR_FORMAT " cached_value=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]", p2i(r->destination()), p2i(r->cached_value()), p2i(r->method_value())); CodeBlob* cb = CodeCache::find_blob(r->destination()); if (cb != nullptr) { - tty->print(" Blob::%s", cb->name()); + st->print(" Blob::%s", cb->name()); } break; } case relocInfo::static_stub_type: { static_stub_Relocation* r = (static_stub_Relocation*) reloc(); - tty->print(" | [static_call=" INTPTR_FORMAT "]", p2i(r->static_call())); + st->print(" | [static_call=" INTPTR_FORMAT "]", p2i(r->static_call())); break; } case relocInfo::trampoline_stub_type: { trampoline_stub_Relocation* r = (trampoline_stub_Relocation*) reloc(); - tty->print(" | [trampoline owner=" INTPTR_FORMAT "]", p2i(r->owner())); + st->print(" | [trampoline owner=" INTPTR_FORMAT "]", p2i(r->owner())); break; } case relocInfo::opt_virtual_call_type: { opt_virtual_call_Relocation* r = (opt_virtual_call_Relocation*) reloc(); - tty->print(" | [destination=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]", + st->print(" | [destination=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]", p2i(r->destination()), p2i(r->method_value())); CodeBlob* cb = CodeCache::find_blob(r->destination()); if (cb != nullptr) { - tty->print(" Blob::%s", cb->name()); + st->print(" Blob::%s", cb->name()); } break; } default: break; } - tty->cr(); + st->cr(); } -void RelocIterator::print() { +void RelocIterator::print_on(outputStream* st) { RelocIterator save_this = (*this); relocInfo* scan = _current; if (!has_current()) scan += 1; // nothing to scan here! @@ -969,32 +996,37 @@ void RelocIterator::print() { got_next = (skip_next || next()); skip_next = false; - tty->print(" @" INTPTR_FORMAT ": ", p2i(scan)); + st->print(" @" INTPTR_FORMAT ": ", p2i(scan)); relocInfo* newscan = _current+1; if (!has_current()) newscan -= 1; // nothing to scan here! while (scan < newscan) { - tty->print("%04x", *(short*)scan & 0xFFFF); + st->print("%04x", *(short*)scan & 0xFFFF); scan++; } - tty->cr(); + st->cr(); if (!got_next) break; - print_current(); + print_current_on(st); } (*this) = save_this; } +//--------------------------------------------------------------------------------- +// Non-product code + +#ifndef PRODUCT + // For the debugger: extern "C" void print_blob_locs(nmethod* nm) { nm->print(); RelocIterator iter(nm); - iter.print(); + iter.print_on(tty); } extern "C" void print_buf_locs(CodeBuffer* cb) { FlagSetting fs(PrintRelocations, true); - cb->print(); + cb->print_on(tty); } #endif // !PRODUCT diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index a10f653bbb1..714a964b28d 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -451,6 +451,8 @@ class relocInfo { length_limit = 1 + 1 + (3*BytesPerWord/BytesPerShort) + 1, have_format = format_width > 0 }; + + static const char* type_name(relocInfo::relocType t); }; #define FORWARD_DECLARE_EACH_CLASS(name) \ @@ -600,6 +602,7 @@ class RelocIterator : public StackObj { // constructor RelocIterator(nmethod* nm, address begin = nullptr, address limit = nullptr); RelocIterator(CodeSection* cb, address begin = nullptr, address limit = nullptr); + RelocIterator(CodeBlob* cb); // get next reloc info, return !eos bool next() { @@ -638,11 +641,11 @@ class RelocIterator : public StackObj { bool addr_in_const() const; address section_start(int n) const { - assert(_section_start[n], "must be initialized"); + assert(_section_start[n], "section %d must be initialized", n); return _section_start[n]; } address section_end(int n) const { - assert(_section_end[n], "must be initialized"); + assert(_section_end[n], "section %d must be initialized", n); return _section_end[n]; } @@ -658,11 +661,9 @@ class RelocIterator : public StackObj { // generic relocation accessor; switches on type to call the above Relocation* reloc(); -#ifndef PRODUCT public: - void print(); - void print_current(); -#endif + void print_on(outputStream* st); + void print_current_on(outputStream* st); }; @@ -672,6 +673,7 @@ class RelocIterator : public StackObj { class Relocation { friend class RelocIterator; + friend class AOTCodeReader; private: // When a relocation has been created by a RelocIterator, @@ -1377,6 +1379,8 @@ class internal_word_Relocation : public DataRelocation { void unpack_data() override; void fix_relocation_after_move(const CodeBuffer* src, CodeBuffer* dest) override; + void fix_relocation_after_aot_load(address orig_base_addr, address current_base_addr); + address target(); // if _target==nullptr, fetch addr from code stream int section() { return _section; } address value() override { return target(); } diff --git a/src/hotspot/share/compiler/oopMap.cpp b/src/hotspot/share/compiler/oopMap.cpp index 88249fc8555..aeb6ac3828e 100644 --- a/src/hotspot/share/compiler/oopMap.cpp +++ b/src/hotspot/share/compiler/oopMap.cpp @@ -729,7 +729,6 @@ bool ImmutableOopMap::has_any(OopMapValue::oop_types type) const { return false; } -#ifdef ASSERT int ImmutableOopMap::nr_of_bytes() const { OopMapStream oms(this); @@ -738,7 +737,6 @@ int ImmutableOopMap::nr_of_bytes() const { } return sizeof(ImmutableOopMap) + oms.stream_position(); } -#endif ImmutableOopMapBuilder::ImmutableOopMapBuilder(const OopMapSet* set) : _set(set), _empty(nullptr), _last(nullptr), _empty_offset(-1), _last_offset(-1), _offset(0), _required(-1), _new_set(nullptr) { _mapping = NEW_RESOURCE_ARRAY(Mapping, _set->size()); diff --git a/src/hotspot/share/compiler/oopMap.hpp b/src/hotspot/share/compiler/oopMap.hpp index 191996f0f05..4b962444738 100644 --- a/src/hotspot/share/compiler/oopMap.hpp +++ b/src/hotspot/share/compiler/oopMap.hpp @@ -292,9 +292,7 @@ public: bool has_derived_oops() const { return _has_derived_oops; } bool has_any(OopMapValue::oop_types type) const; -#ifdef ASSERT - int nr_of_bytes() const; // this is an expensive operation, only used in debug builds -#endif + int nr_of_bytes() const; // this is an expensive operation, only used in debug builds or in aot code generation void oops_do(const frame* fr, const RegisterMap* reg_map, OopClosure* f, DerivedOopClosure* df) const; void oops_do(const frame* fr, const RegisterMap* reg_map, OopClosure* f, DerivedPointerIterationMode derived_mode) const; @@ -378,9 +376,7 @@ class OopMapStream : public StackObj { bool is_done() { if(!_valid_omv) { find_next(); } return !_valid_omv; } void next() { find_next(); } OopMapValue current() { return _omv; } -#ifdef ASSERT int stream_position() const { return _stream.position(); } -#endif }; class ImmutableOopMapBuilder { diff --git a/src/hotspot/share/include/cds.h b/src/hotspot/share/include/cds.h index 8819e4c00be..685fba47349 100644 --- a/src/hotspot/share/include/cds.h +++ b/src/hotspot/share/include/cds.h @@ -35,12 +35,12 @@ // // Also, this is a C header file. Do not use C++ here. -#define NUM_CDS_REGIONS 4 // this must be the same as MetaspaceShared::n_regions +#define NUM_CDS_REGIONS 5 // this must be the same as MetaspaceShared::n_regions #define CDS_ARCHIVE_MAGIC 0xf00baba2 #define CDS_DYNAMIC_ARCHIVE_MAGIC 0xf00baba8 #define CDS_PREIMAGE_ARCHIVE_MAGIC 0xcafea07c #define CDS_GENERIC_HEADER_SUPPORTED_MIN_VERSION 13 -#define CURRENT_CDS_ARCHIVE_VERSION 18 +#define CURRENT_CDS_ARCHIVE_VERSION 19 typedef struct CDSFileMapRegion { int _crc; // CRC checksum of this region. diff --git a/src/hotspot/share/memory/allocation.hpp b/src/hotspot/share/memory/allocation.hpp index b67dcd43e4d..93fe4f2e99b 100644 --- a/src/hotspot/share/memory/allocation.hpp +++ b/src/hotspot/share/memory/allocation.hpp @@ -314,7 +314,9 @@ class MetaspaceObj { f(ConstantPoolCache) \ f(Annotations) \ f(MethodCounters) \ - f(RecordComponent) + f(RecordComponent) \ + f(AdapterHandlerEntry) \ + f(AdapterFingerPrint) #define METASPACE_OBJ_TYPE_DECLARE(name) name ## Type, #define METASPACE_OBJ_TYPE_NAME_CASE(name) case name ## Type: return #name; diff --git a/src/hotspot/share/nmt/virtualMemoryTracker.cpp b/src/hotspot/share/nmt/virtualMemoryTracker.cpp index 9e5d0a18c10..e6fac91f245 100644 --- a/src/hotspot/share/nmt/virtualMemoryTracker.cpp +++ b/src/hotspot/share/nmt/virtualMemoryTracker.cpp @@ -396,6 +396,11 @@ bool VirtualMemoryTracker::add_reserved_region(address base_addr, size_t size, return true; } + if (reserved_rgn->mem_tag() == mtCode) { + assert(reserved_rgn->contain_region(base_addr, size), "Reserved code region should contain this mapping region"); + return true; + } + // Print some more details. Don't use UL here to avoid circularities. tty->print_cr("Error: existing region: [" INTPTR_FORMAT "-" INTPTR_FORMAT "), memory tag %u.\n" " new region: [" INTPTR_FORMAT "-" INTPTR_FORMAT "), memory tag %u.", diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index 07f2c559604..ce7d77a38cc 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -31,6 +31,7 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmClasses.hpp" +#include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" #include "code/debugInfoRec.hpp" #include "compiler/compilationPolicy.hpp" @@ -392,6 +393,7 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { } else { it->push(&_constMethod); } + it->push(&_adapter); it->push(&_method_data); it->push(&_method_counters); NOT_PRODUCT(it->push(&_name);) @@ -405,11 +407,18 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { void Method::remove_unshareable_info() { unlink_method(); + if (AOTCodeCache::is_dumping_adapters() && _adapter != nullptr) { + _adapter->remove_unshareable_info(); + } JFR_ONLY(REMOVE_METHOD_ID(this);) } void Method::restore_unshareable_info(TRAPS) { assert(is_method() && is_valid_method(this), "ensure C++ vtable is restored"); + if (_adapter != nullptr) { + assert(_adapter->is_linked(), "must be"); + _from_compiled_entry = _adapter->get_c2i_entry(); + } assert(!queued_for_compilation(), "method's queued_for_compilation flag should not be set"); } #endif @@ -1137,7 +1146,9 @@ void Method::unlink_code() { void Method::unlink_method() { assert(CDSConfig::is_dumping_archive(), "sanity"); _code = nullptr; - _adapter = nullptr; + if (!AOTCodeCache::is_dumping_adapters() || AdapterHandlerLibrary::is_abstract_method_adapter(_adapter)) { + _adapter = nullptr; + } _i2i_entry = nullptr; _from_compiled_entry = nullptr; _from_interpreted_entry = nullptr; @@ -1178,14 +1189,18 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { // If the code cache is full, we may reenter this function for the // leftover methods that weren't linked. if (adapter() != nullptr) { - return; + if (adapter()->is_shared()) { + assert(adapter()->is_linked(), "Adapter is shared but not linked"); + } else { + return; + } } assert( _code == nullptr, "nothing compiled yet" ); // Setup interpreter entrypoint assert(this == h_method(), "wrong h_method()" ); - assert(adapter() == nullptr, "init'd to null"); + assert(adapter() == nullptr || adapter()->is_linked(), "init'd to null or restored from cache"); address entry = Interpreter::entry_for_method(h_method); assert(entry != nullptr, "interpreter entry must be non-null"); // Sets both _i2i_entry and _from_interpreted_entry @@ -1206,7 +1221,10 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { // called from the vtable. We need adapters on such methods that get loaded // later. Ditto for mega-morphic itable calls. If this proves to be a // problem we'll make these lazily later. - (void) make_adapters(h_method, CHECK); + if (_adapter == nullptr) { + (void) make_adapters(h_method, CHECK); + assert(adapter()->is_linked(), "Adapter must have been linked"); + } // ONLY USE the h_method now as make_adapter may have blocked @@ -1482,6 +1500,9 @@ methodHandle Method::make_method_handle_intrinsic(vmIntrinsics::ID iid, #if INCLUDE_CDS void Method::restore_archived_method_handle_intrinsic(methodHandle m, TRAPS) { + if (m->adapter() != nullptr) { + m->set_from_compiled_entry(m->adapter()->get_c2i_entry()); + } m->link_method(m, CHECK); if (m->intrinsic_id() == vmIntrinsics::_linkToNative) { diff --git a/src/hotspot/share/opto/runtime.hpp b/src/hotspot/share/opto/runtime.hpp index fd7fc796aa4..633cced0090 100644 --- a/src/hotspot/share/opto/runtime.hpp +++ b/src/hotspot/share/opto/runtime.hpp @@ -113,6 +113,7 @@ enum class OptoStubId :int { class OptoRuntime : public AllStatic { friend class Matcher; // allow access to stub names + friend class AOTCodeAddressTable; private: // declare opto stub address/blob holder static fields diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index 3756cdd4fd2..48e01f82025 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -24,6 +24,7 @@ #include "classfile/stringTable.hpp" #include "classfile/symbolTable.hpp" +#include "code/aotCodeCache.hpp" #include "compiler/compiler_globals.hpp" #include "gc/shared/collectedHeap.hpp" #include "gc/shared/gcHeapSummary.hpp" @@ -143,7 +144,7 @@ jint init_globals() { LSAN_REGISTER_ROOT_REGION(summary.start(), summary.reserved_size()); } #endif // LEAK_SANITIZER - + AOTCodeCache::init2(); // depends on universe_init AsyncLogWriter::initialize(); gc_barrier_stubs_init(); // depends on universe_init, must be before interpreter_init continuations_init(); // must precede continuation stub generation @@ -156,6 +157,8 @@ jint init_globals() { InterfaceSupport_init(); VMRegImpl::set_regName(); // need this before generate_stubs (for printing oop maps). SharedRuntime::generate_stubs(); + AOTCodeCache::init_shared_blobs_table(); // need this after generate_stubs + SharedRuntime::init_adapter_library(); // do this after AOTCodeCache::init_shared_blobs_table return JNI_OK; } diff --git a/src/hotspot/share/runtime/mutexLocker.cpp b/src/hotspot/share/runtime/mutexLocker.cpp index faed0d5f952..8cb239636d2 100644 --- a/src/hotspot/share/runtime/mutexLocker.cpp +++ b/src/hotspot/share/runtime/mutexLocker.cpp @@ -123,6 +123,8 @@ Mutex* CodeHeapStateAnalytics_lock = nullptr; Mutex* ExternalsRecorder_lock = nullptr; +Mutex* AOTCodeCStrings_lock = nullptr; + Monitor* ContinuationRelativize_lock = nullptr; Mutex* Metaspace_lock = nullptr; @@ -319,6 +321,8 @@ void mutex_init() { // tty_lock is held when printing nmethod and its relocations which use this lock. MUTEX_DEFL(ExternalsRecorder_lock , PaddedMutex , tty_lock); + MUTEX_DEFL(AOTCodeCStrings_lock , PaddedMutex , tty_lock); + MUTEX_DEFL(Threads_lock , PaddedMonitor, CompileThread_lock, true); MUTEX_DEFL(Compile_lock , PaddedMutex , MethodCompileQueue_lock); MUTEX_DEFL(JNICritical_lock , PaddedMonitor, AdapterHandlerLibrary_lock); // used for JNI critical regions diff --git a/src/hotspot/share/runtime/mutexLocker.hpp b/src/hotspot/share/runtime/mutexLocker.hpp index 3bd01b575c4..c307f70ba80 100644 --- a/src/hotspot/share/runtime/mutexLocker.hpp +++ b/src/hotspot/share/runtime/mutexLocker.hpp @@ -148,6 +148,8 @@ extern Mutex* CodeHeapStateAnalytics_lock; // lock print functions against extern Mutex* ExternalsRecorder_lock; // used to guard access to the external addresses table +extern Mutex* AOTCodeCStrings_lock; // used to guard access to the AOT code C strings table + extern Monitor* ContinuationRelativize_lock; #if INCLUDE_JVMCI diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 3695d9dfe76..733e8a78d90 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -22,11 +22,14 @@ * */ +#include "cds/archiveBuilder.hpp" +#include "cds/archiveUtils.inline.hpp" #include "classfile/classLoader.hpp" #include "classfile/javaClasses.inline.hpp" #include "classfile/stringTable.hpp" #include "classfile/vmClasses.hpp" #include "classfile/vmSymbols.hpp" +#include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" #include "code/compiledIC.hpp" #include "code/nmethod.inline.hpp" @@ -147,8 +150,6 @@ void SharedRuntime::generate_stubs() { generate_throw_exception(SharedStubId::throw_NullPointerException_at_call_id, CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call)); - AdapterHandlerLibrary::initialize(); - #if COMPILER2_OR_JVMCI // Vectors are generated only by C2 and JVMCI. bool support_wide = is_wide_vector(MaxVectorSize); @@ -168,6 +169,10 @@ void SharedRuntime::generate_stubs() { generate_deopt_blob(); } +void SharedRuntime::init_adapter_library() { + AdapterHandlerLibrary::initialize(); +} + #if INCLUDE_JFR //------------------------------generate jfr runtime stubs ------ void SharedRuntime::generate_jfr_stubs() { @@ -2170,13 +2175,14 @@ void SharedRuntime::print_call_statistics(uint64_t comp_total) { #ifndef PRODUCT static int _lookups; // number of calls to lookup static int _equals; // number of buckets checked with matching hash -static int _hits; // number of successful lookups +static int _archived_hits; // number of successful lookups in archived table +static int _runtime_hits; // number of successful lookups in runtime table static int _compact; // number of equals calls with compact signature #endif // A simple wrapper class around the calling convention information // that allows sharing of adapters for the same calling convention. -class AdapterFingerPrint : public CHeapObj { +class AdapterFingerPrint : public MetaspaceObj { private: enum { _basic_type_bits = 4, @@ -2187,12 +2193,29 @@ class AdapterFingerPrint : public CHeapObj { // TO DO: Consider integrating this with a more global scheme for compressing signatures. // For now, 4 bits per components (plus T_VOID gaps after double/long) is not excessive. - union { - int _compact[_compact_int_count]; - int* _fingerprint; - } _value; - int _length; // A negative length indicates the fingerprint is in the compact form, - // Otherwise _value._fingerprint is the array. + int _length; + int _value[_compact_int_count]; + + // Private construtor. Use allocate() to get an instance. + AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) { + // Pack the BasicTypes with 8 per int + _length = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; + int sig_index = 0; + for (int index = 0; index < _length; index++) { + int value = 0; + for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) { + int bt = adapter_encoding(sig_bt[sig_index++]); + assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); + value = (value << _basic_type_bits) | bt; + } + _value[index] = value; + } + } + + // Call deallocate instead + ~AdapterFingerPrint() { + FreeHeap(this); + } // Remap BasicTypes that are handled equivalently by the adapters. // These are correct for the current system but someday it might be @@ -2229,64 +2252,63 @@ class AdapterFingerPrint : public CHeapObj { } } - public: - AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) { - // The fingerprint is based on the BasicType signature encoded - // into an array of ints with eight entries per int. - int* ptr; - int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; - if (len <= _compact_int_count) { - assert(_compact_int_count == 3, "else change next line"); - _value._compact[0] = _value._compact[1] = _value._compact[2] = 0; - // Storing the signature encoded as signed chars hits about 98% - // of the time. - _length = -len; - ptr = _value._compact; - } else { - _length = len; - _value._fingerprint = NEW_C_HEAP_ARRAY(int, _length, mtCode); - ptr = _value._fingerprint; - } + void* operator new(size_t size, size_t fp_size) throw() { + assert(fp_size >= size, "sanity check"); + void* p = AllocateHeap(fp_size, mtCode); + memset(p, 0, fp_size); + return p; + } - // Now pack the BasicTypes with 8 per int - int sig_index = 0; - for (int index = 0; index < len; index++) { - int value = 0; - for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) { - int bt = adapter_encoding(sig_bt[sig_index++]); - assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); - value = (value << _basic_type_bits) | bt; + template + void iterate_args(Function function) { + for (int i = 0; i < length(); i++) { + unsigned val = (unsigned)value(i); + // args are packed so that first/lower arguments are in the highest + // bits of each int value, so iterate from highest to the lowest + for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { + unsigned v = (val >> j) & _basic_type_mask; + if (v == 0) { + continue; + } + function(v); } - ptr[index] = value; } } - ~AdapterFingerPrint() { - if (_length > 0) { - FREE_C_HEAP_ARRAY(int, _value._fingerprint); - } + public: + static int allocation_size(int total_args_passed, BasicType* sig_bt) { + int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; + return sizeof(AdapterFingerPrint) + (len > _compact_int_count ? (len - _compact_int_count) * sizeof(int) : 0); + } + + static AdapterFingerPrint* allocate(int total_args_passed, BasicType* sig_bt) { + int size_in_bytes = allocation_size(total_args_passed, sig_bt); + return new (size_in_bytes) AdapterFingerPrint(total_args_passed, sig_bt); + } + + static void deallocate(AdapterFingerPrint* fp) { + fp->~AdapterFingerPrint(); } int value(int index) { - if (_length < 0) { - return _value._compact[index]; - } - return _value._fingerprint[index]; + return _value[index]; } + int length() { if (_length < 0) return -_length; return _length; } bool is_compact() { - return _length <= 0; + return _length <= _compact_int_count; } unsigned int compute_hash() { int hash = 0; for (int i = 0; i < length(); i++) { int v = value(i); - hash = (hash << 8) ^ v ^ (hash >> 5); + //Add arithmetic operation to the hash, like +3 to improve hashing + hash = ((hash << 8) ^ v ^ (hash >> 5)) + 3; } return (unsigned int)hash; } @@ -2300,59 +2322,90 @@ class AdapterFingerPrint : public CHeapObj { return st.as_string(); } -#ifndef PRODUCT - // Reconstitutes the basic type arguments from the fingerprint, - // producing strings like LIJDF const char* as_basic_args_string() { stringStream st; bool long_prev = false; - for (int i = 0; i < length(); i++) { - unsigned val = (unsigned)value(i); - // args are packed so that first/lower arguments are in the highest - // bits of each int value, so iterate from highest to the lowest - for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { - unsigned v = (val >> j) & _basic_type_mask; - if (v == 0) { - assert(i == length() - 1, "Only expect zeroes in the last word"); - continue; - } - if (long_prev) { - long_prev = false; - if (v == T_VOID) { - st.print("J"); - } else { - st.print("L"); - } - } - switch (v) { - case T_INT: st.print("I"); break; - case T_LONG: long_prev = true; break; - case T_FLOAT: st.print("F"); break; - case T_DOUBLE: st.print("D"); break; - case T_VOID: break; - default: ShouldNotReachHere(); + iterate_args([&] (int arg) { + if (long_prev) { + long_prev = false; + if (arg == T_VOID) { + st.print("J"); + } else { + st.print("L"); } } - } + switch (arg) { + case T_INT: st.print("I"); break; + case T_LONG: long_prev = true; break; + case T_FLOAT: st.print("F"); break; + case T_DOUBLE: st.print("D"); break; + case T_VOID: break; + default: ShouldNotReachHere(); + } + }); if (long_prev) { st.print("L"); } return st.as_string(); } -#endif // !product + + BasicType* as_basic_type(int& nargs) { + nargs = 0; + GrowableArray btarray; + bool long_prev = false; + + iterate_args([&] (int arg) { + if (long_prev) { + long_prev = false; + if (arg == T_VOID) { + btarray.append(T_LONG); + } else { + btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter + } + } + switch (arg) { + case T_INT: // fallthrough + case T_FLOAT: // fallthrough + case T_DOUBLE: + case T_VOID: + btarray.append((BasicType)arg); + break; + case T_LONG: + long_prev = true; + break; + default: ShouldNotReachHere(); + } + }); + + if (long_prev) { + btarray.append(T_OBJECT); + } + + nargs = btarray.length(); + BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, nargs); + int index = 0; + GrowableArrayIterator iter = btarray.begin(); + while (iter != btarray.end()) { + sig_bt[index++] = *iter; + ++iter; + } + assert(index == btarray.length(), "sanity check"); +#ifdef ASSERT + { + AdapterFingerPrint* compare_fp = AdapterFingerPrint::allocate(nargs, sig_bt); + assert(this->equals(compare_fp), "sanity check"); + AdapterFingerPrint::deallocate(compare_fp); + } +#endif + return sig_bt; + } bool equals(AdapterFingerPrint* other) { if (other->_length != _length) { return false; - } - if (_length < 0) { - assert(_compact_int_count == 3, "else change next line"); - return _value._compact[0] == other->_value._compact[0] && - _value._compact[1] == other->_value._compact[1] && - _value._compact[2] == other->_value._compact[2]; } else { for (int i = 0; i < _length; i++) { - if (_value._fingerprint[i] != other->_value._fingerprint[i]) { + if (_value[i] != other->_value[i]) { return false; } } @@ -2360,6 +2413,11 @@ class AdapterFingerPrint : public CHeapObj { return true; } + // methods required by virtue of being a MetaspaceObj + void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ } + int size() const { return (int)heap_word_size(sizeof(AdapterFingerPrint) + (_length > _compact_int_count ? (_length - _compact_int_count) * sizeof(int) : 0)); } + MetaspaceObj::Type type() const { return AdapterFingerPrintType; } + static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) { NOT_PRODUCT(_equals++); return fp1->equals(fp2); @@ -2370,27 +2428,63 @@ class AdapterFingerPrint : public CHeapObj { } }; +#if INCLUDE_CDS +static inline bool adapter_fp_equals_compact_hashtable_entry(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) { + return AdapterFingerPrint::equals(entry->fingerprint(), fp); +} + +class ArchivedAdapterTable : public OffsetCompactHashtable< + AdapterFingerPrint*, + AdapterHandlerEntry*, + adapter_fp_equals_compact_hashtable_entry> {}; +#endif // INCLUDE_CDS + // A hashtable mapping from AdapterFingerPrints to AdapterHandlerEntries using AdapterHandlerTable = ResourceHashtable; static AdapterHandlerTable* _adapter_handler_table; +static GrowableArray* _adapter_handler_list = nullptr; // Find a entry with the same fingerprint if it exists -static AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt) { +AdapterHandlerEntry* AdapterHandlerLibrary::lookup(int total_args_passed, BasicType* sig_bt) { NOT_PRODUCT(_lookups++); assert_lock_strong(AdapterHandlerLibrary_lock); - AdapterFingerPrint fp(total_args_passed, sig_bt); - AdapterHandlerEntry** entry = _adapter_handler_table->get(&fp); - if (entry != nullptr) { + AdapterFingerPrint* fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + AdapterHandlerEntry* entry = nullptr; +#if INCLUDE_CDS + // if we are building the archive then the archived adapter table is + // not valid and we need to use the ones added to the runtime table + if (!AOTCodeCache::is_dumping_adapters()) { + // Search archived table first. It is read-only table so can be searched without lock + entry = _aot_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */); + if (entry != nullptr) { #ifndef PRODUCT - if (fp.is_compact()) _compact++; - _hits++; + if (fp->is_compact()) { + _compact++; + } + _archived_hits++; #endif - return *entry; + } } - return nullptr; +#endif // INCLUDE_CDS + if (entry == nullptr) { + assert_lock_strong(AdapterHandlerLibrary_lock); + AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp); + if (entry_p != nullptr) { + entry = *entry_p; + assert(entry->fingerprint()->equals(fp), "fingerprint mismatch key fp %s %s (hash=%d) != found fp %s %s (hash=%d)", + entry->fingerprint()->as_basic_args_string(), entry->fingerprint()->as_string(), entry->fingerprint()->compute_hash(), + fp->as_basic_args_string(), fp->as_string(), fp->compute_hash()); + #ifndef PRODUCT + if (fp->is_compact()) _compact++; + _runtime_hits++; + #endif + } + } + AdapterFingerPrint::deallocate(fp); + return entry; } #ifndef PRODUCT @@ -2402,8 +2496,9 @@ static void print_table_statistics() { ts.print(tty, "AdapterHandlerTable"); tty->print_cr("AdapterHandlerTable (table_size=%d, entries=%d)", _adapter_handler_table->table_size(), _adapter_handler_table->number_of_entries()); - tty->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d compact %d", - _lookups, _equals, _hits, _compact); + int total_hits = _archived_hits + _runtime_hits; + tty->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d (archived=%d+runtime=%d) compact %d", + _lookups, _equals, total_hits, _archived_hits, _runtime_hits, _compact); } #endif @@ -2415,10 +2510,14 @@ AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr; -const int AdapterHandlerLibrary_size = 16*K; +#if INCLUDE_CDS +ArchivedAdapterTable AdapterHandlerLibrary::_aot_adapter_handler_table; +#endif // INCLUDE_CDS +static const int AdapterHandlerLibrary_size = 16*K; BufferBlob* AdapterHandlerLibrary::_buffer = nullptr; BufferBlob* AdapterHandlerLibrary::buffer_blob() { + assert(_buffer != nullptr, "should be initialized"); return _buffer; } @@ -2441,7 +2540,39 @@ static void post_adapter_creation(const AdapterBlob* new_adapter, } } +void AdapterHandlerLibrary::create_abstract_method_handler() { + assert_lock_strong(AdapterHandlerLibrary_lock); + // Create a special handler for abstract methods. Abstract methods + // are never compiled so an i2c entry is somewhat meaningless, but + // throw AbstractMethodError just in case. + // Pass wrong_method_abstract for the c2i transitions to return + // AbstractMethodError for invalid invocations. + address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub(); + _abstract_method_handler = AdapterHandlerLibrary::new_entry(AdapterFingerPrint::allocate(0, nullptr)); + _abstract_method_handler->set_entry_points(SharedRuntime::throw_AbstractMethodError_entry(), + wrong_method_abstract, + wrong_method_abstract, + nullptr); +} + void AdapterHandlerLibrary::initialize() { + { + ResourceMark rm; + MutexLocker mu(AdapterHandlerLibrary_lock); + _adapter_handler_table = new (mtCode) AdapterHandlerTable(); + _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size); + create_abstract_method_handler(); + } + +#if INCLUDE_CDS + // Link adapters in AOT Cache to their code in AOT Code Cache + if (!_aot_adapter_handler_table.empty()) { + link_aot_adapters(); + lookup_simple_adapters(); + return; + } +#endif // INCLUDE_CDS + ResourceMark rm; AdapterBlob* no_arg_blob = nullptr; AdapterBlob* int_arg_blob = nullptr; @@ -2449,39 +2580,27 @@ void AdapterHandlerLibrary::initialize() { AdapterBlob* obj_int_arg_blob = nullptr; AdapterBlob* obj_obj_arg_blob = nullptr; { - _adapter_handler_table = new (mtCode) AdapterHandlerTable(); MutexLocker mu(AdapterHandlerLibrary_lock); - // Create a special handler for abstract methods. Abstract methods - // are never compiled so an i2c entry is somewhat meaningless, but - // throw AbstractMethodError just in case. - // Pass wrong_method_abstract for the c2i transitions to return - // AbstractMethodError for invalid invocations. - address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub(); - _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(0, nullptr), - SharedRuntime::throw_AbstractMethodError_entry(), - wrong_method_abstract, wrong_method_abstract); - - _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size); - _no_arg_handler = create_adapter(no_arg_blob, 0, nullptr, true); + _no_arg_handler = create_adapter(no_arg_blob, 0, nullptr); BasicType obj_args[] = { T_OBJECT }; - _obj_arg_handler = create_adapter(obj_arg_blob, 1, obj_args, true); + _obj_arg_handler = create_adapter(obj_arg_blob, 1, obj_args); BasicType int_args[] = { T_INT }; - _int_arg_handler = create_adapter(int_arg_blob, 1, int_args, true); + _int_arg_handler = create_adapter(int_arg_blob, 1, int_args); BasicType obj_int_args[] = { T_OBJECT, T_INT }; - _obj_int_arg_handler = create_adapter(obj_int_arg_blob, 2, obj_int_args, true); + _obj_int_arg_handler = create_adapter(obj_int_arg_blob, 2, obj_int_args); BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT }; - _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, 2, obj_obj_args, true); + _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, 2, obj_obj_args); assert(no_arg_blob != nullptr && - obj_arg_blob != nullptr && - int_arg_blob != nullptr && - obj_int_arg_blob != nullptr && - obj_obj_arg_blob != nullptr, "Initial adapters must be properly created"); + obj_arg_blob != nullptr && + int_arg_blob != nullptr && + obj_int_arg_blob != nullptr && + obj_obj_arg_blob != nullptr, "Initial adapters must be properly created"); } // Outside of the lock @@ -2492,14 +2611,8 @@ void AdapterHandlerLibrary::initialize() { post_adapter_creation(obj_obj_arg_blob, _obj_obj_arg_handler); } -AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint, - address i2c_entry, - address c2i_entry, - address c2i_unverified_entry, - address c2i_no_clinit_check_entry) { - // Insert an entry into the table - return new AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, - c2i_no_clinit_check_entry); +AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint) { + return AdapterHandlerEntry::allocate(fingerprint); } AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) { @@ -2583,6 +2696,22 @@ class AdapterSignatureIterator : public SignatureIterator { } }; + +const char* AdapterHandlerEntry::_entry_names[] = { + "i2c", "c2i", "c2i_unverified", "c2i_no_clinit_check" +}; + +#ifdef ASSERT +void AdapterHandlerLibrary::verify_adapter_sharing(int total_args_passed, BasicType* sig_bt, AdapterHandlerEntry* cached_entry) { + AdapterBlob* comparison_blob = nullptr; + AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, total_args_passed, sig_bt, true); + assert(comparison_blob == nullptr, "no blob should be created when creating an adapter for comparison"); + assert(comparison_entry->compare_code(cached_entry), "code must match"); + // Release the one just created + AdapterHandlerEntry::deallocate(comparison_entry); +} +#endif /* ASSERT*/ + AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) { // Use customized signature handler. Need to lock around updates to // the _adapter_handler_table (it is not safe for concurrent readers @@ -2596,7 +2725,7 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth } ResourceMark rm; - AdapterBlob* new_adapter = nullptr; + AdapterBlob* adapter_blob = nullptr; // Fill in the signature array, for the calling-convention call. int total_args_passed = method->size_of_parameters(); // All args on stack @@ -2612,111 +2741,292 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth entry = lookup(total_args_passed, sig_bt); if (entry != nullptr) { + assert(entry->is_linked(), "AdapterHandlerEntry must have been linked"); #ifdef ASSERT - if (VerifyAdapterSharing) { - AdapterBlob* comparison_blob = nullptr; - AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, total_args_passed, sig_bt, false); - assert(comparison_blob == nullptr, "no blob should be created when creating an adapter for comparison"); - assert(comparison_entry->compare_code(entry), "code must match"); - // Release the one just created and return the original - delete comparison_entry; + if (!entry->is_shared() && VerifyAdapterSharing) { + verify_adapter_sharing(total_args_passed, sig_bt, entry); } #endif - return entry; + } else { + entry = create_adapter(adapter_blob, total_args_passed, sig_bt); } - - entry = create_adapter(new_adapter, total_args_passed, sig_bt, /* allocate_code_blob */ true); } // Outside of the lock - if (new_adapter != nullptr) { - post_adapter_creation(new_adapter, entry); + if (adapter_blob != nullptr) { + post_adapter_creation(adapter_blob, entry); } return entry; } -AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter, - int total_args_passed, - BasicType* sig_bt, - bool allocate_code_blob) { +AdapterBlob* AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* handler) { + ResourceMark rm; + const char* name = AdapterHandlerLibrary::name(handler->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(handler->fingerprint()); + int offsets[AdapterHandlerEntry::ENTRIES_COUNT]; + + AdapterBlob* adapter_blob = nullptr; + CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::Adapter, id, name, AdapterHandlerEntry::ENTRIES_COUNT, offsets); + if (blob != nullptr) { + adapter_blob = blob->as_adapter_blob(); + address i2c_entry = adapter_blob->content_begin(); + assert(offsets[0] == 0, "sanity check"); + handler->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + } + return adapter_blob; +} + +#ifndef PRODUCT +void AdapterHandlerLibrary::print_adapter_handler_info(outputStream* st, AdapterHandlerEntry* handler, AdapterBlob* adapter_blob) { + ttyLocker ttyl; + ResourceMark rm; + int insts_size = adapter_blob->code_size(); + handler->print_adapter_on(tty); + st->print_cr("i2c argument handler for: %s %s (%d bytes generated)", + handler->fingerprint()->as_basic_args_string(), + handler->fingerprint()->as_string(), insts_size); + st->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(handler->get_c2i_entry())); + if (Verbose || PrintStubCode) { + address first_pc = handler->base_address(); + if (first_pc != nullptr) { + Disassembler::decode(first_pc, first_pc + insts_size, st, &adapter_blob->asm_remarks()); + st->cr(); + } + } +} +#endif // PRODUCT + +bool AdapterHandlerLibrary::generate_adapter_code(AdapterBlob*& adapter_blob, + AdapterHandlerEntry* handler, + int total_args_passed, + BasicType* sig_bt, + bool is_transient) { if (log_is_enabled(Info, perf, class, link)) { ClassLoader::perf_method_adapters_count()->inc(); } - // StubRoutines::_final_stubs_code is initialized after this function can be called. As a result, - // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated prior - // to all StubRoutines::_final_stubs_code being set. Checks refer to runtime range checks generated - // in an I2C stub that ensure that an I2C stub is called from an interpreter frame or stubs. - bool contains_all_checks = StubRoutines::final_stubs_code() != nullptr; - + BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache + CodeBuffer buffer(buf); + short buffer_locs[20]; + buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, + sizeof(buffer_locs)/sizeof(relocInfo)); + MacroAssembler masm(&buffer); VMRegPair stack_regs[16]; VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); - BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache - CodeBuffer buffer(buf); - short buffer_locs[20]; - buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, - sizeof(buffer_locs)/sizeof(relocInfo)); - - // Make a C heap allocated version of the fingerprint to store in the adapter - AdapterFingerPrint* fingerprint = new AdapterFingerPrint(total_args_passed, sig_bt); - MacroAssembler _masm(&buffer); - AdapterHandlerEntry* entry = SharedRuntime::generate_i2c2i_adapters(&_masm, - total_args_passed, - comp_args_on_stack, - sig_bt, - regs, - fingerprint); - + SharedRuntime::generate_i2c2i_adapters(&masm, + total_args_passed, + comp_args_on_stack, + sig_bt, + regs, + handler); #ifdef ASSERT if (VerifyAdapterSharing) { - entry->save_code(buf->code_begin(), buffer.insts_size()); - if (!allocate_code_blob) { - return entry; + handler->save_code(buf->code_begin(), buffer.insts_size()); + if (is_transient) { + return true; } } #endif - new_adapter = AdapterBlob::create(&buffer); - NOT_PRODUCT(int insts_size = buffer.insts_size()); - if (new_adapter == nullptr) { + adapter_blob = AdapterBlob::create(&buffer); + if (adapter_blob == nullptr) { // CodeCache is full, disable compilation // Ought to log this but compile log is only per compile thread // and we're some non descript Java thread. - return nullptr; + return false; } - entry->relocate(new_adapter->content_begin()); + if (!is_transient && AOTCodeCache::is_dumping_adapters()) { + // try to save generated code + const char* name = AdapterHandlerLibrary::name(handler->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(handler->fingerprint()); + int entry_offset[AdapterHandlerEntry::ENTRIES_COUNT]; + assert(AdapterHandlerEntry::ENTRIES_COUNT == 4, "sanity"); + address i2c_entry = handler->get_i2c_entry(); + entry_offset[0] = 0; // i2c_entry offset + entry_offset[1] = handler->get_c2i_entry() - i2c_entry; + entry_offset[2] = handler->get_c2i_unverified_entry() - i2c_entry; + entry_offset[3] = handler->get_c2i_no_clinit_check_entry() - i2c_entry; + bool success = AOTCodeCache::store_code_blob(*adapter_blob, AOTCodeEntry::Adapter, id, name, AdapterHandlerEntry::ENTRIES_COUNT, entry_offset); + assert(success || !AOTCodeCache::is_dumping_adapters(), "caching of adapter must be disabled"); + } + handler->relocate(adapter_blob->content_begin()); #ifndef PRODUCT // debugging support if (PrintAdapterHandlers || PrintStubCode) { - ttyLocker ttyl; - entry->print_adapter_on(tty); - tty->print_cr("i2c argument handler #%d for: %s %s (%d bytes generated)", - _adapter_handler_table->number_of_entries(), fingerprint->as_basic_args_string(), - fingerprint->as_string(), insts_size); - tty->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(entry->get_c2i_entry())); - if (Verbose || PrintStubCode) { - address first_pc = entry->base_address(); - if (first_pc != nullptr) { - Disassembler::decode(first_pc, first_pc + insts_size, tty - NOT_PRODUCT(COMMA &new_adapter->asm_remarks())); - tty->cr(); - } - } + print_adapter_handler_info(tty, handler, adapter_blob); } #endif - - // Add the entry only if the entry contains all required checks (see sharedRuntime_xxx.cpp) - // The checks are inserted only if -XX:+VerifyAdapterCalls is specified. - if (contains_all_checks || !VerifyAdapterCalls) { - assert_lock_strong(AdapterHandlerLibrary_lock); - _adapter_handler_table->put(fingerprint, entry); - } - return entry; + return true; } +AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& adapter_blob, + int total_args_passed, + BasicType* sig_bt, + bool is_transient) { + AdapterFingerPrint* fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + AdapterHandlerEntry* handler = AdapterHandlerLibrary::new_entry(fp); + if (!generate_adapter_code(adapter_blob, handler, total_args_passed, sig_bt, is_transient)) { + AdapterHandlerEntry::deallocate(handler); + return nullptr; + } + if (!is_transient) { + assert_lock_strong(AdapterHandlerLibrary_lock); + _adapter_handler_table->put(fp, handler); + } + return handler; +} + +#if INCLUDE_CDS +void AdapterHandlerEntry::remove_unshareable_info() { +#ifdef ASSERT + _saved_code = nullptr; + _saved_code_length = 0; +#endif // ASSERT + set_entry_points(nullptr, nullptr, nullptr, nullptr, false); +} + +class CopyAdapterTableToArchive : StackObj { +private: + CompactHashtableWriter* _writer; + ArchiveBuilder* _builder; +public: + CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer), + _builder(ArchiveBuilder::current()) + {} + + bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { + LogStreamHandle(Trace, cds) lsh; + if (ArchiveBuilder::current()->has_been_archived((address)entry)) { + assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be"); + AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp); + assert(buffered_fp != nullptr,"sanity check"); + AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry); + assert(buffered_entry != nullptr,"sanity check"); + + uint hash = fp->compute_hash(); + u4 delta = _builder->buffer_to_offset_u4((address)buffered_entry); + _writer->add(hash, delta); + if (lsh.is_enabled()) { + address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); + address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); + log_trace(cds)("Added fp=%p (%s), entry=%p to the archived adater table", buffered_fp, buffered_fp->as_basic_args_string(), buffered_entry); + } + } else { + if (lsh.is_enabled()) { + log_trace(cds)("Skipping adapter handler %p (fp=%s) as it is not archived", entry, fp->as_basic_args_string()); + } + } + return true; + } +}; + +void AdapterHandlerLibrary::dump_aot_adapter_table() { + CompactHashtableStats stats; + CompactHashtableWriter writer(_adapter_handler_table->number_of_entries(), &stats); + CopyAdapterTableToArchive copy(&writer); + _adapter_handler_table->iterate(©); + writer.dump(&_aot_adapter_handler_table, "archived adapter table"); +} + +void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) { + _aot_adapter_handler_table.serialize_header(soc); +} + +AdapterBlob* AdapterHandlerLibrary::link_aot_adapter_handler(AdapterHandlerEntry* handler) { +#ifdef ASSERT + if (TestAOTAdapterLinkFailure) { + return nullptr; + } +#endif + AdapterBlob* blob = lookup_aot_cache(handler); +#ifndef PRODUCT + // debugging support + if ((blob != nullptr) && (PrintAdapterHandlers || PrintStubCode)) { + print_adapter_handler_info(tty, handler, blob); + } +#endif + return blob; +} + +// This method is used during production run to link archived adapters (stored in AOT Cache) +// to their code in AOT Code Cache +void AdapterHandlerEntry::link() { + AdapterBlob* adapter_blob = nullptr; + ResourceMark rm; + assert(_fingerprint != nullptr, "_fingerprint must not be null"); + bool generate_code = false; + // Generate code only if AOTCodeCache is not available, or + // caching adapters is disabled, or we fail to link + // the AdapterHandlerEntry to its code in the AOTCodeCache + if (AOTCodeCache::is_using_adapters()) { + adapter_blob = AdapterHandlerLibrary::link_aot_adapter_handler(this); + if (adapter_blob == nullptr) { + log_warning(cds)("Failed to link AdapterHandlerEntry (fp=%s) to its code in the AOT code cache", _fingerprint->as_basic_args_string()); + generate_code = true; + } + } else { + generate_code = true; + } + if (generate_code) { + int nargs; + BasicType* bt = _fingerprint->as_basic_type(nargs); + if (!AdapterHandlerLibrary::generate_adapter_code(adapter_blob, this, nargs, bt, /* is_transient */ false)) { + // Don't throw exceptions during VM initialization because java.lang.* classes + // might not have been initialized, causing problems when constructing the + // Java exception object. + vm_exit_during_initialization("Out of space in CodeCache for adapters"); + } + } + // Outside of the lock + if (adapter_blob != nullptr) { + post_adapter_creation(adapter_blob, this); + } + assert(_linked, "AdapterHandlerEntry must now be linked"); +} + +void AdapterHandlerLibrary::link_aot_adapters() { + _aot_adapter_handler_table.iterate([](AdapterHandlerEntry* entry) { + assert(!entry->is_linked(), "AdapterHandlerEntry is already linked!"); + entry->link(); + }); +} + +// This method is called during production run to lookup simple adapters +// in the archived adapter handler table +void AdapterHandlerLibrary::lookup_simple_adapters() { + assert(!_aot_adapter_handler_table.empty(), "archived adapter handler table is empty"); + + MutexLocker mu(AdapterHandlerLibrary_lock); + _no_arg_handler = lookup(0, nullptr); + + BasicType obj_args[] = { T_OBJECT }; + _obj_arg_handler = lookup(1, obj_args); + + BasicType int_args[] = { T_INT }; + _int_arg_handler = lookup(1, int_args); + + BasicType obj_int_args[] = { T_OBJECT, T_INT }; + _obj_int_arg_handler = lookup(2, obj_int_args); + + BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT }; + _obj_obj_arg_handler = lookup(2, obj_obj_args); + + assert(_no_arg_handler != nullptr && + _obj_arg_handler != nullptr && + _int_arg_handler != nullptr && + _obj_int_arg_handler != nullptr && + _obj_obj_arg_handler != nullptr, "Initial adapters not found in archived adapter handler table"); + assert(_no_arg_handler->is_linked() && + _obj_arg_handler->is_linked() && + _int_arg_handler->is_linked() && + _obj_int_arg_handler->is_linked() && + _obj_obj_arg_handler->is_linked(), "Initial adapters not in linked state"); +} +#endif // INCLUDE_CDS + address AdapterHandlerEntry::base_address() { address base = _i2c_entry; if (base == nullptr) base = _c2i_entry; @@ -2741,12 +3051,24 @@ void AdapterHandlerEntry::relocate(address new_base) { assert(base_address() == new_base, ""); } +void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) { + LogStreamHandle(Trace, cds) lsh; + if (lsh.is_enabled()) { + lsh.print("Iter(AdapterHandlerEntry): %p(%s)", this, _fingerprint->as_basic_args_string()); + lsh.cr(); + } + it->push(&_fingerprint); +} AdapterHandlerEntry::~AdapterHandlerEntry() { - delete _fingerprint; + if (_fingerprint != nullptr) { + AdapterFingerPrint::deallocate(_fingerprint); + _fingerprint = nullptr; + } #ifdef ASSERT FREE_C_HEAP_ARRAY(unsigned char, _saved_code); #endif + FreeHeap(this); } @@ -3051,28 +3373,61 @@ JRT_END bool AdapterHandlerLibrary::contains(const CodeBlob* b) { bool found = false; - auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { - return (found = (b == CodeCache::find_blob(a->get_i2c_entry()))); +#if INCLUDE_CDS + auto findblob_archived_table = [&] (AdapterHandlerEntry* handler) { + return (found = (b == CodeCache::find_blob(handler->get_i2c_entry()))); }; - assert_locked_or_safepoint(AdapterHandlerLibrary_lock); - _adapter_handler_table->iterate(findblob); + _aot_adapter_handler_table.iterate(findblob_archived_table); +#endif // INCLUDE_CDS + if (!found) { + auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { + return (found = (b == CodeCache::find_blob(a->get_i2c_entry()))); + }; + assert_locked_or_safepoint(AdapterHandlerLibrary_lock); + _adapter_handler_table->iterate(findblob_runtime_table); + } return found; } +const char* AdapterHandlerLibrary::name(AdapterFingerPrint* fingerprint) { + return fingerprint->as_basic_args_string(); +} + +uint32_t AdapterHandlerLibrary::id(AdapterFingerPrint* fingerprint) { + unsigned int hash = fingerprint->compute_hash(); + return hash; +} + void AdapterHandlerLibrary::print_handler_on(outputStream* st, const CodeBlob* b) { bool found = false; - auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { - if (b == CodeCache::find_blob(a->get_i2c_entry())) { +#if INCLUDE_CDS + auto findblob_archived_table = [&] (AdapterHandlerEntry* handler) { + if (b == CodeCache::find_blob(handler->get_i2c_entry())) { found = true; st->print("Adapter for signature: "); - a->print_adapter_on(st); + handler->print_adapter_on(st); return true; } else { return false; // keep looking + } }; - assert_locked_or_safepoint(AdapterHandlerLibrary_lock); - _adapter_handler_table->iterate(findblob); + _aot_adapter_handler_table.iterate(findblob_archived_table); +#endif // INCLUDE_CDS + if (!found) { + auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { + if (b == CodeCache::find_blob(a->get_i2c_entry())) { + found = true; + st->print("Adapter for signature: "); + a->print_adapter_on(st); + return true; + } else { + return false; // keep looking + } + }; + assert_locked_or_safepoint(AdapterHandlerLibrary_lock); + _adapter_handler_table->iterate(findblob_runtime_table); + } assert(found, "Should have found handler"); } @@ -3101,6 +3456,13 @@ void AdapterHandlerLibrary::print_statistics() { #endif /* PRODUCT */ +bool AdapterHandlerLibrary::is_abstract_method_adapter(AdapterHandlerEntry* entry) { + if (entry == _abstract_method_handler) { + return true; + } + return false; +} + JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current)) assert(current == JavaThread::current(), "pre-condition"); StackOverflow* overflow_state = current->stack_overflow_state(); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 94cbc460cad..de16a33fdcc 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -25,10 +25,12 @@ #ifndef SHARE_RUNTIME_SHAREDRUNTIME_HPP #define SHARE_RUNTIME_SHAREDRUNTIME_HPP +#include "classfile/compactHashtable.hpp" #include "code/codeBlob.hpp" #include "code/vmreg.hpp" #include "interpreter/linkResolver.hpp" #include "memory/allStatic.hpp" +#include "memory/metaspaceClosure.hpp" #include "memory/resourceArea.hpp" #include "runtime/stubDeclarations.hpp" #include "utilities/macros.hpp" @@ -114,6 +116,7 @@ class SharedRuntime: AllStatic { // For c2: call to runtime to return a buffer lease. static RuntimeStub* generate_jfr_return_lease(); #endif + static void init_adapter_library(); static const char *stub_name(SharedStubId id) { assert(id > SharedStubId::NO_STUBID && id < SharedStubId::NUM_STUBIDS, "stub id out of range"); @@ -464,12 +467,12 @@ class SharedRuntime: AllStatic { // pointer as needed. This means the i2c adapter code doesn't need any special // handshaking path with compiled code to keep the stack walking correct. - static AdapterHandlerEntry* generate_i2c2i_adapters(MacroAssembler *_masm, - int total_args_passed, - int max_arg, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint); + static void generate_i2c2i_adapters(MacroAssembler *_masm, + int total_args_passed, + int max_arg, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler); static void gen_i2c_adapter(MacroAssembler *_masm, int total_args_passed, @@ -666,15 +669,21 @@ class SharedRuntime: AllStatic { // used by the adapters. The code generation happens here because it's very // similar to what the adapters have to do. -class AdapterHandlerEntry : public CHeapObj { +class AdapterHandlerEntry : public MetaspaceObj { friend class AdapterHandlerLibrary; + public: + static const int ENTRIES_COUNT = 4; + private: AdapterFingerPrint* _fingerprint; address _i2c_entry; address _c2i_entry; address _c2i_unverified_entry; address _c2i_no_clinit_check_entry; + bool _linked; + + static const char *_entry_names[]; #ifdef ASSERT // Captures code and signature used to generate this adapter when @@ -683,27 +692,58 @@ class AdapterHandlerEntry : public CHeapObj { int _saved_code_length; #endif - AdapterHandlerEntry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, - address c2i_unverified_entry, - address c2i_no_clinit_check_entry) : + AdapterHandlerEntry(AdapterFingerPrint* fingerprint) : _fingerprint(fingerprint), - _i2c_entry(i2c_entry), - _c2i_entry(c2i_entry), - _c2i_unverified_entry(c2i_unverified_entry), - _c2i_no_clinit_check_entry(c2i_no_clinit_check_entry) + _i2c_entry(nullptr), + _c2i_entry(nullptr), + _c2i_unverified_entry(nullptr), + _c2i_no_clinit_check_entry(nullptr), + _linked(false) #ifdef ASSERT - , _saved_code_length(0) + , _saved_code(nullptr), + _saved_code_length(0) #endif { } ~AdapterHandlerEntry(); + // Allocate on CHeap instead of metaspace (see JDK-8331086). + // Dummy argument is used to avoid C++ warning about using + // deleted opearator MetaspaceObj::delete(). + void* operator new(size_t size, size_t dummy) throw() { + void* p = AllocateHeap(size, mtCode); + memset(p, 0, size); + return p; + } + public: + static AdapterHandlerEntry* allocate(AdapterFingerPrint* fingerprint) { + return new(0) AdapterHandlerEntry(fingerprint); + } + + static void deallocate(AdapterHandlerEntry *handler) { + handler->~AdapterHandlerEntry(); + } + + void set_entry_points(address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry, bool linked = true) { + _i2c_entry = i2c_entry; + _c2i_entry = c2i_entry; + _c2i_unverified_entry = c2i_unverified_entry; + _c2i_no_clinit_check_entry = c2i_no_clinit_check_entry; + _linked = linked; + } + address get_i2c_entry() const { return _i2c_entry; } address get_c2i_entry() const { return _c2i_entry; } address get_c2i_unverified_entry() const { return _c2i_unverified_entry; } address get_c2i_no_clinit_check_entry() const { return _c2i_no_clinit_check_entry; } + static const char* entry_name(int i) { + assert(i >=0 && i < ENTRIES_COUNT, "entry id out of range"); + return _entry_names[i]; + } + + bool is_linked() const { return _linked; } address base_address(); void relocate(address new_base); @@ -717,8 +757,19 @@ class AdapterHandlerEntry : public CHeapObj { //virtual void print_on(outputStream* st) const; DO NOT USE void print_adapter_on(outputStream* st) const; + + void metaspace_pointers_do(MetaspaceClosure* it); + int size() const {return (int)heap_word_size(sizeof(AdapterHandlerEntry)); } + MetaspaceObj::Type type() const { return AdapterHandlerEntryType; } + + void remove_unshareable_info() NOT_CDS_RETURN; + void link() NOT_CDS_RETURN; }; +#if INCLUDE_CDS +class ArchivedAdapterTable; +#endif // INCLUDE_CDS + class AdapterHandlerLibrary: public AllStatic { friend class SharedRuntime; private: @@ -729,31 +780,54 @@ class AdapterHandlerLibrary: public AllStatic { static AdapterHandlerEntry* _obj_arg_handler; static AdapterHandlerEntry* _obj_int_arg_handler; static AdapterHandlerEntry* _obj_obj_arg_handler; +#if INCLUDE_CDS + static ArchivedAdapterTable _aot_adapter_handler_table; +#endif // INCLUDE_CDS static BufferBlob* buffer_blob(); static void initialize(); + static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); + static AdapterBlob* lookup_aot_cache(AdapterHandlerEntry* handler); static AdapterHandlerEntry* create_adapter(AdapterBlob*& new_adapter, int total_args_passed, BasicType* sig_bt, - bool allocate_code_blob); - static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); + bool is_transient = false); + static void create_abstract_method_handler(); + static void lookup_simple_adapters() NOT_CDS_RETURN; +#ifndef PRODUCT + static void print_adapter_handler_info(outputStream* st, AdapterHandlerEntry* handler, AdapterBlob* adapter_blob); +#endif // PRODUCT public: - static AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, - address i2c_entry, - address c2i_entry, - address c2i_unverified_entry, - address c2i_no_clinit_check_entry = nullptr); + static AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint); static void create_native_wrapper(const methodHandle& method); static AdapterHandlerEntry* get_adapter(const methodHandle& method); + static AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt); + static bool generate_adapter_code(AdapterBlob*& adapter_blob, + AdapterHandlerEntry* handler, + int total_args_passed, + BasicType* sig_bt, + bool is_transient); + +#ifdef ASSERT + static void verify_adapter_sharing(int total_args_passed, BasicType* sig_bt, AdapterHandlerEntry* cached); +#endif // ASSERT static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); static bool contains(const CodeBlob* b); + static const char* name(AdapterFingerPrint* fingerprint); + static uint32_t id(AdapterFingerPrint* fingerprint); #ifndef PRODUCT static void print_statistics(); #endif // PRODUCT + static bool is_abstract_method_adapter(AdapterHandlerEntry* adapter); + + static AdapterBlob* link_aot_adapter_handler(AdapterHandlerEntry* handler) NOT_CDS_RETURN_(nullptr); + static void dump_aot_adapter_table() NOT_CDS_RETURN; + static void serialize_shared_table_header(SerializeClosure* soc) NOT_CDS_RETURN; + static void link_aot_adapters() NOT_CDS_RETURN; }; #endif // SHARE_RUNTIME_SHAREDRUNTIME_HPP diff --git a/test/hotspot/jtreg/TEST.groups b/test/hotspot/jtreg/TEST.groups index a990de30ad4..c52cc84f177 100644 --- a/test/hotspot/jtreg/TEST.groups +++ b/test/hotspot/jtreg/TEST.groups @@ -410,6 +410,7 @@ hotspot_appcds_dynamic = \ runtime/cds/appcds/ \ -runtime/cds/appcds/aotCache \ -runtime/cds/appcds/aotClassLinking \ + -runtime/cds/appcds/aotCode \ -runtime/cds/appcds/applications \ -runtime/cds/appcds/cacheObject \ -runtime/cds/appcds/complexURI \ @@ -511,6 +512,7 @@ hotspot_aot_classlinking = \ runtime/cds \ -runtime/cds/appcds/aotCache \ -runtime/cds/appcds/aotClassLinking \ + -runtime/cds/appcds/aotCode \ -runtime/cds/appcds/BadBSM.java \ -runtime/cds/appcds/cacheObject/ArchivedIntegerCacheTest.java \ -runtime/cds/appcds/cacheObject/ArchivedModuleCompareTest.java \ @@ -589,6 +591,7 @@ hotspot_metaspace = \ # A subset of AppCDS tests to be run in tier1 tier1_runtime_appcds = \ runtime/cds/appcds/aotCache/HelloAOTCache.java \ + runtime/cds/appcds/aotCode \ runtime/cds/appcds/HelloTest.java tier1_runtime_appcds_exclude = \ diff --git a/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java new file mode 100644 index 00000000000..24479535057 --- /dev/null +++ b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +/** + * @test + * @summary Sanity test of combinations of the AOT Code Caching diagnostic flags + * @requires vm.cds + * @requires vm.cds.supports.aot.class.linking + * @requires vm.flagless + * @comment work around JDK-8345635 + * @requires !vm.jvmci.enabled + * @library /test/lib /test/setup_aot + * @build AOTCodeFlags JavacBenchApp + * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar app.jar + * JavacBenchApp + * JavacBenchApp$ClassFile + * JavacBenchApp$FileManager + * JavacBenchApp$SourceFile + * @run driver AOTCodeFlags + */ + +import jdk.test.lib.cds.CDSAppTester; +import jdk.test.lib.process.OutputAnalyzer; + +public class AOTCodeFlags { + public static int flag_sign = 0; + public static void main(String... args) throws Exception { + Tester t = new Tester(); + for (int i = 0; i < 2; i++) { + flag_sign = i; + t.run(new String[] {"AOT"}); + } + } + static class Tester extends CDSAppTester { + public Tester() { + super("AOTCodeFlags" + flag_sign); + } + + @Override + public String classpath(RunMode runMode) { + return "app.jar"; + } + + @Override + public String[] vmArgs(RunMode runMode) { + switch (runMode) { + case RunMode.ASSEMBLY: + case RunMode.PRODUCTION: + return new String[] { + "-XX:+UnlockDiagnosticVMOptions", + "-XX:" + (flag_sign == 0 ? "-" : "+") + "AOTAdapterCaching", + "-Xlog:aot+codecache+init=debug", + "-Xlog:aot+codecache+exit=debug", + }; + } + return new String[] {}; + } + + @Override + public String[] appCommandLine(RunMode runMode) { + return new String[] { + "JavacBenchApp", "10" + }; + } + + @Override + public void checkExecution(OutputAnalyzer out, RunMode runMode) throws Exception { + if (flag_sign == 0) { + switch (runMode) { + case RunMode.ASSEMBLY: + case RunMode.PRODUCTION: + out.shouldNotContain("Adapters: total"); + break; + } + + } else { + switch (runMode) { + case RunMode.ASSEMBLY: + case RunMode.PRODUCTION: + out.shouldContain("Adapters: total"); + break; + } + } + } + + } +}