diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp index 3cdc9b70d52..3317ccc3b53 100644 --- a/src/hotspot/cpu/riscv/assembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp @@ -961,81 +961,239 @@ protected: #undef INSN -enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11}; + enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11}; -#define INSN(NAME, op, funct3, funct7) \ - void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { \ - unsigned insn = 0; \ - patch((address)&insn, 6, 0, op); \ - patch((address)&insn, 14, 12, funct3); \ - patch_reg((address)&insn, 7, Rd); \ - patch_reg((address)&insn, 15, Rs1); \ - patch_reg((address)&insn, 20, Rs2); \ - patch((address)&insn, 31, 27, funct7); \ - patch((address)&insn, 26, 25, memory_order); \ - emit(insn); \ + private: + + enum AmoWidthFunct3 : uint8_t { + AMO_WIDTH_BYTE = 0b000, // Zabha extension + AMO_WIDTH_HALFWORD = 0b001, // Zabha extension + AMO_WIDTH_WORD = 0b010, + AMO_WIDTH_DOUBLEWORD = 0b011, + AMO_WIDTH_QUADWORD = 0b100, + // 0b101 to 0b111 are reserved + }; + + enum AmoOperationFunct5 : uint8_t { + AMO_ADD = 0b00000, + AMO_SWAP = 0b00001, + AMO_LR = 0b00010, + AMO_SC = 0b00011, + AMO_XOR = 0b00100, + AMO_OR = 0b01000, + AMO_AND = 0b01100, + AMO_MIN = 0b10000, + AMO_MAX = 0b10100, + AMO_MINU = 0b11000, + AMO_MAXU = 0b11100, + AMO_CAS = 0b00101 // Zacas + }; + + static constexpr uint32_t OP_AMO_MAJOR = 0b0101111; + + template + void amo_base(Register Rd, Register Rs1, uint8_t Rs2, Aqrl memory_order = aqrl) { + assert(width > AMO_WIDTH_HALFWORD || UseZabha, "Must be"); + assert(funct5 != AMO_CAS || UseZacas, "Must be"); + unsigned insn = 0; + patch((address)&insn, 6, 0, OP_AMO_MAJOR); + patch_reg((address)&insn, 7, Rd); + patch((address)&insn, 14, 12, width); + patch_reg((address)&insn, 15, Rs1); + patch((address)&insn, 24, 20, Rs2); + patch((address)&insn, 26, 25, memory_order); + patch((address)&insn, 31, 27, funct5); + emit(insn); } - INSN(amoswap_w, 0b0101111, 0b010, 0b00001); - INSN(amoadd_w, 0b0101111, 0b010, 0b00000); - INSN(amoxor_w, 0b0101111, 0b010, 0b00100); - INSN(amoand_w, 0b0101111, 0b010, 0b01100); - INSN(amoor_w, 0b0101111, 0b010, 0b01000); - INSN(amomin_w, 0b0101111, 0b010, 0b10000); - INSN(amomax_w, 0b0101111, 0b010, 0b10100); - INSN(amominu_w, 0b0101111, 0b010, 0b11000); - INSN(amomaxu_w, 0b0101111, 0b010, 0b11100); - INSN(amoswap_d, 0b0101111, 0b011, 0b00001); - INSN(amoadd_d, 0b0101111, 0b011, 0b00000); - INSN(amoxor_d, 0b0101111, 0b011, 0b00100); - INSN(amoand_d, 0b0101111, 0b011, 0b01100); - INSN(amoor_d, 0b0101111, 0b011, 0b01000); - INSN(amomin_d, 0b0101111, 0b011, 0b10000); - INSN(amomax_d , 0b0101111, 0b011, 0b10100); - INSN(amominu_d, 0b0101111, 0b011, 0b11000); - INSN(amomaxu_d, 0b0101111, 0b011, 0b11100); - INSN(amocas_w, 0b0101111, 0b010, 0b00101); - INSN(amocas_d, 0b0101111, 0b011, 0b00101); -#undef INSN - -enum operand_size { int8, int16, int32, uint32, int64 }; - -#define INSN(NAME, op, funct3, funct7) \ - void NAME(Register Rd, Register Rs1, Aqrl memory_order = relaxed) { \ - unsigned insn = 0; \ - uint32_t val = memory_order & 0x3; \ - patch((address)&insn, 6, 0, op); \ - patch((address)&insn, 14, 12, funct3); \ - patch_reg((address)&insn, 7, Rd); \ - patch_reg((address)&insn, 15, Rs1); \ - patch((address)&insn, 25, 20, 0b00000); \ - patch((address)&insn, 31, 27, funct7); \ - patch((address)&insn, 26, 25, val); \ - emit(insn); \ + template + void amo_base(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2->raw_encoding(), memory_order); } - INSN(lr_w, 0b0101111, 0b010, 0b00010); - INSN(lr_d, 0b0101111, 0b011, 0b00010); + public: -#undef INSN - -#define INSN(NAME, op, funct3, funct7) \ - void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = relaxed) { \ - unsigned insn = 0; \ - uint32_t val = memory_order & 0x3; \ - patch((address)&insn, 6, 0, op); \ - patch((address)&insn, 14, 12, funct3); \ - patch_reg((address)&insn, 7, Rd); \ - patch_reg((address)&insn, 15, Rs2); \ - patch_reg((address)&insn, 20, Rs1); \ - patch((address)&insn, 31, 27, funct7); \ - patch((address)&insn, 26, 25, val); \ - emit(insn); \ + void amoadd_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); } - INSN(sc_w, 0b0101111, 0b010, 0b00011); - INSN(sc_d, 0b0101111, 0b011, 0b00011); -#undef INSN + void amoadd_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoadd_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoadd_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoswap_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoswap_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoswap_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoswap_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoxor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoxor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoxor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoxor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoand_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoand_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoand_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amoand_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomin_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomin_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomin_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomin_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amominu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amominu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amominu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amominu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomax_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomax_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomax_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomax_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomaxu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomaxu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomaxu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amomaxu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + protected: + + void lr_w(Register Rd, Register Rs1, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, 0, memory_order); + } + + void lr_d(Register Rd, Register Rs1, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, 0, memory_order); + } + + void sc_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void sc_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amocas_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amocas_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amocas_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + void amocas_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { + amo_base(Rd, Rs1, Rs2, memory_order); + } + + public: + + enum operand_size { int8, int16, int32, uint32, int64 }; // Immediate Instruction #define INSN(NAME, op, funct3) \ diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp index 3ef084d30fc..d67e05bbb6d 100644 --- a/src/hotspot/cpu/riscv/globals_riscv.hpp +++ b/src/hotspot/cpu/riscv/globals_riscv.hpp @@ -107,6 +107,7 @@ define_pd_global(intx, InlineSmallCode, 1000); product(bool, UseZfh, false, DIAGNOSTIC, "Use Zfh instructions") \ product(bool, UseZfhmin, false, DIAGNOSTIC, "Use Zfhmin instructions") \ product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \ + product(bool, UseZabha, false, EXPERIMENTAL, "Use UseZabha instructions") \ product(bool, UseZcb, false, EXPERIMENTAL, "Use Zcb instructions") \ product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions") \ product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \ diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index bc6bf88d0ad..c755d9ae23d 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -3798,7 +3798,7 @@ void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register o void MacroAssembler::load_reserved(Register dst, Register addr, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire) { switch (size) { case int64: @@ -3819,15 +3819,15 @@ void MacroAssembler::load_reserved(Register dst, void MacroAssembler::store_conditional(Register dst, Register new_val, Register addr, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl release) { switch (size) { case int64: - sc_d(dst, new_val, addr, release); + sc_d(dst, addr, new_val, release); break; case int32: case uint32: - sc_w(dst, new_val, addr, release); + sc_w(dst, addr, new_val, release); break; default: ShouldNotReachHere(); @@ -3836,7 +3836,7 @@ void MacroAssembler::store_conditional(Register dst, void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Register shift, Register mask, Register aligned_addr) { assert(size == int8 || size == int16, "unsupported operand size"); @@ -3866,10 +3866,11 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte // which are forced to work with 4-byte aligned address. void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, bool result_as_bool, Register tmp1, Register tmp2, Register tmp3) { + assert(!(UseZacas && UseZabha), "Use amocas"); assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1); Register scratch0 = t0, aligned_addr = t1; @@ -3902,13 +3903,13 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, notr(scratch1, mask); bind(retry); - lr_w(result, aligned_addr, acquire); + load_reserved(result, aligned_addr, operand_size::int32, acquire); andr(scratch0, result, mask); bne(scratch0, expected, fail); andr(scratch0, result, scratch1); // scratch1 is ~mask orr(scratch0, scratch0, new_val); - sc_w(scratch0, scratch0, aligned_addr, release); + store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release); bnez(scratch0, retry); } @@ -3940,10 +3941,11 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, // failed. void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, Register tmp1, Register tmp2, Register tmp3) { + assert(!(UseZacas && UseZabha), "Use amocas"); assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1); Register scratch0 = t0, aligned_addr = t1; @@ -3974,13 +3976,13 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected, } else { notr(scratch1, mask); - lr_w(result, aligned_addr, acquire); + load_reserved(result, aligned_addr, operand_size::int32, acquire); andr(scratch0, result, mask); bne(scratch0, expected, fail); andr(scratch0, result, scratch1); // scratch1 is ~mask orr(scratch0, scratch0, new_val); - sc_w(scratch0, scratch0, aligned_addr, release); + store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release); bnez(scratch0, fail); } @@ -3997,10 +3999,10 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected, void MacroAssembler::cmpxchg(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, bool result_as_bool) { - assert(size != int8 && size != int16, "unsupported operand size"); + assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size"); assert_different_registers(addr, t0); assert_different_registers(expected, t0); assert_different_registers(new_val, t0); @@ -4058,10 +4060,10 @@ void MacroAssembler::cmpxchg(Register addr, Register expected, void MacroAssembler::weak_cmpxchg(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result) { - + assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size"); assert_different_registers(addr, t0); assert_different_registers(expected, t0); assert_different_registers(new_val, t0); @@ -4134,7 +4136,7 @@ ATOMIC_XCHGU(xchgalwu, xchgalw) #undef ATOMIC_XCHGU void MacroAssembler::atomic_cas(Register prev, Register newv, Register addr, - enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) { + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) { switch (size) { case int64: amocas_d(prev, addr, newv, (Assembler::Aqrl)(acquire | release)); @@ -4146,6 +4148,12 @@ void MacroAssembler::atomic_cas(Register prev, Register newv, Register addr, amocas_w(prev, addr, newv, (Assembler::Aqrl)(acquire | release)); zext(prev, prev, 32); break; + case int16: + amocas_h(prev, addr, newv, (Assembler::Aqrl)(acquire | release)); + break; + case int8: + amocas_b(prev, addr, newv, (Assembler::Aqrl)(acquire | release)); + break; default: ShouldNotReachHere(); } diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp index 6f80a02ddc6..f302b3602ad 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp @@ -1187,26 +1187,26 @@ public: void cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, Label &succeed, Label *fail); void cmpxchg(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, bool result_as_bool = false); void weak_cmpxchg(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result); void cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Register shift, Register mask, Register aligned_addr); void cmpxchg_narrow_value(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, bool result_as_bool, Register tmp1, Register tmp2, Register tmp3); void weak_cmpxchg_narrow_value(Register addr, Register expected, Register new_val, - enum operand_size size, + Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, Register tmp1, Register tmp2, Register tmp3); @@ -1223,7 +1223,7 @@ public: void atomic_xchgwu(Register prev, Register newv, Register addr); void atomic_xchgalwu(Register prev, Register newv, Register addr); - void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size, + void atomic_cas(Register prev, Register newv, Register addr, Assembler::operand_size size, Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed); // Emit a far call/jump. Only invalidates the tmp register which @@ -1636,8 +1636,8 @@ private: int bitset_to_regs(unsigned int bitset, unsigned char* regs); Address add_memory_helper(const Address dst, Register tmp); - void load_reserved(Register dst, Register addr, enum operand_size size, Assembler::Aqrl acquire); - void store_conditional(Register dst, Register new_val, Register addr, enum operand_size size, Assembler::Aqrl release); + void load_reserved(Register dst, Register addr, Assembler::operand_size size, Assembler::Aqrl acquire); + void store_conditional(Register dst, Register new_val, Register addr, Assembler::operand_size size, Assembler::Aqrl release); public: void lightweight_lock(Register basic_lock, Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow); diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index c711d2db640..e838ee184fb 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -2304,42 +2304,6 @@ encode %{ } %} - enc_class riscv_enc_cmpxchgw(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ - __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, - /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, - /*result as bool*/ true); - %} - - enc_class riscv_enc_cmpxchgn(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ - __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, - /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, - /*result as bool*/ true); - %} - - enc_class riscv_enc_cmpxchg(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{ - __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, - /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, - /*result as bool*/ true); - %} - - enc_class riscv_enc_cmpxchgw_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ - __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, - /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, - /*result as bool*/ true); - %} - - enc_class riscv_enc_cmpxchgn_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ - __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, - /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, - /*result as bool*/ true); - %} - - enc_class riscv_enc_cmpxchg_acq(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{ - __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, - /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, - /*result as bool*/ true); - %} - // compare and branch instruction encodings enc_class riscv_enc_j(label lbl) %{ @@ -5250,18 +5214,20 @@ instruct prefetchalloc( memory mem ) %{ // standard CompareAndSwapX when we are using barriers // these have higher priority than the rules selected by a predicate -instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndSwapB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) %{ + predicate(!UseZabha || !UseZacas); + match(Set res (CompareAndSwapB mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); format %{ "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t" - "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB_narrow" %} ins_encode %{ @@ -5273,18 +5239,42 @@ instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R1 ins_pipe(pipe_slow); %} -instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ + predicate(UseZabha && UseZacas); + + match(Set res (CompareAndSwapB mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(!UseZabha || !UseZacas); + match(Set res (CompareAndSwapS mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); format %{ "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t" - "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS_narrow" %} ins_encode %{ @@ -5296,18 +5286,44 @@ instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R1 ins_pipe(pipe_slow); %} +instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate(UseZabha && UseZacas); + + match(Set res (CompareAndSwapS mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */); + %} + + ins_pipe(pipe_slow); +%} + instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ match(Set res (CompareAndSwapI mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapI" %} - ins_encode(riscv_enc_cmpxchgw(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} @@ -5316,14 +5332,18 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval %{ match(Set res (CompareAndSwapL mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapL" %} - ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} @@ -5334,14 +5354,18 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval match(Set res (CompareAndSwapP mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapP" %} - ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} @@ -5349,35 +5373,40 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapN" %} - ins_encode(riscv_enc_cmpxchgn(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} // alternative CompareAndSwapX when we are eliding barriers -instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndSwapBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n)); match(Set res (CompareAndSwapB mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t" - "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq_narrow" %} ins_encode %{ @@ -5389,20 +5418,42 @@ instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI ins_pipe(pipe_slow); %} -instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndSwapB mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n)); match(Set res (CompareAndSwapS mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t" - "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq_narrow" %} ins_encode %{ @@ -5414,20 +5465,46 @@ instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI ins_pipe(pipe_slow); %} +instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndSwapS mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */); + %} + + ins_pipe(pipe_slow); +%} + instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ predicate(needs_acquiring_load_reserved(n)); match(Set res (CompareAndSwapI mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapIAcq" %} - ins_encode(riscv_enc_cmpxchgw_acq(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} @@ -5438,14 +5515,18 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL new match(Set res (CompareAndSwapL mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapLAcq" %} - ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} @@ -5456,14 +5537,18 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new match(Set res (CompareAndSwapP mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapPAcq" %} - ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} @@ -5474,14 +5559,18 @@ instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN new match(Set res (CompareAndSwapN mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t" "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapNAcq" %} - ins_encode(riscv_enc_cmpxchgn_acq(res, mem, oldval, newval)); + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} ins_pipe(pipe_slow); %} @@ -5492,17 +5581,19 @@ instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN new // no trailing StoreLoad barrier emitted by C2. Unfortunately we // can't check the type of memory ordering here, so we always emit a // sc_d(w) with rl bit set. -instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndExchangeB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) %{ + predicate(!UseZabha || !UseZacas); + match(Set res (CompareAndExchangeB mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB" + "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB_narrow" %} ins_encode %{ @@ -5514,17 +5605,39 @@ instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg ins_pipe(pipe_slow); %} -instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ + predicate(UseZabha && UseZacas); + + match(Set res (CompareAndExchangeB mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(!UseZabha || !UseZacas); + match(Set res (CompareAndExchangeS mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS" + "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS_narrow" %} ins_encode %{ @@ -5536,13 +5649,31 @@ instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg ins_pipe(pipe_slow); %} +instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate(UseZabha && UseZacas); + + match(Set res (CompareAndExchangeS mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ match(Set res (CompareAndExchangeI mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeI" @@ -5560,9 +5691,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne %{ match(Set res (CompareAndExchangeL mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeL" @@ -5579,11 +5708,10 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval) %{ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 3); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeN" @@ -5600,11 +5728,10 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeP" @@ -5618,19 +5745,19 @@ instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP ne ins_pipe(pipe_slow); %} -instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndExchangeBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n)); match(Set res (CompareAndExchangeB mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ - "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq" + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq_narrow" %} ins_encode %{ @@ -5642,19 +5769,39 @@ instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i ins_pipe(pipe_slow); %} -instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndExchangeB mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n)); match(Set res (CompareAndExchangeS mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ - "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq" + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq_narrow" %} ins_encode %{ @@ -5666,15 +5813,33 @@ instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i ins_pipe(pipe_slow); %} +instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndExchangeS mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ predicate(needs_acquiring_load_reserved(n)); match(Set res (CompareAndExchangeI mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeIAcq" @@ -5694,9 +5859,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL match(Set res (CompareAndExchangeL mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeLAcq" @@ -5716,9 +5879,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN match(Set res (CompareAndExchangeN mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeNAcq" @@ -5738,9 +5899,7 @@ instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP match(Set res (CompareAndExchangeP mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); - - effect(TEMP_DEF res); + ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangePAcq" @@ -5754,18 +5913,20 @@ instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP ins_pipe(pipe_slow); %} -instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct weakCompareAndSwapB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) %{ + predicate(!UseZabha || !UseZacas); + match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ "weak_cmpxchg $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t" - "# $res == 1 when success, #@weakCompareAndSwapB" + "# $res == 1 when success, #@weakCompareAndSwapB_narrow" %} ins_encode %{ @@ -5777,18 +5938,41 @@ instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg ins_pipe(pipe_slow); %} -instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ + predicate(UseZabha && UseZacas); + + match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "weak_cmpxchg $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapB" + %} + + ins_encode %{ + __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(!UseZabha || !UseZacas); + match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ "weak_cmpxchg $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t" - "# $res == 1 when success, #@weakCompareAndSwapS" + "# $res == 1 when success, #@weakCompareAndSwapS_narrow" %} ins_encode %{ @@ -5800,11 +5984,32 @@ instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg ins_pipe(pipe_slow); %} +instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate(UseZabha && UseZacas); + + match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "weak_cmpxchg $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapS" + %} + + ins_encode %{ + __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t" @@ -5823,7 +6028,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne %{ match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t" @@ -5841,9 +6046,10 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t" @@ -5861,9 +6067,10 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval) %{ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t" @@ -5878,20 +6085,20 @@ instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne ins_pipe(pipe_slow); %} -instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct weakCompareAndSwapBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n)); match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t" - "# $res == 1 when success, #@weakCompareAndSwapBAcq" + "# $res == 1 when success, #@weakCompareAndSwapBAcq_narrow" %} ins_encode %{ @@ -5903,20 +6110,41 @@ instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i ins_pipe(pipe_slow); %} -instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, - iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n)); + + match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "weak_cmpxchg_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapBAcq" + %} + + ins_encode %{ + __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n)); match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7); + ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); format %{ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t" - "# $res == 1 when success, #@weakCompareAndSwapSAcq" + "# $res == 1 when success, #@weakCompareAndSwapSAcq_narrow" %} ins_encode %{ @@ -5928,13 +6156,34 @@ instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i ins_pipe(pipe_slow); %} +instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n)); + + match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ + "weak_cmpxchg_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapSAcq" + %} + + ins_encode %{ + __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) %{ predicate(needs_acquiring_load_reserved(n)); match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t" @@ -5955,7 +6204,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t" @@ -5976,7 +6225,7 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t" @@ -5997,7 +6246,7 @@ instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); - ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + ins_cost(2 * VOLATILE_REF_COST); format %{ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t" diff --git a/test/hotspot/gtest/riscv/test_assembler_riscv.cpp b/test/hotspot/gtest/riscv/test_assembler_riscv.cpp index 45f0ed390bd..c6754eacae6 100644 --- a/test/hotspot/gtest/riscv/test_assembler_riscv.cpp +++ b/test/hotspot/gtest/riscv/test_assembler_riscv.cpp @@ -107,6 +107,14 @@ TEST_VM(RiscV, cmov) { } } +template +bool using_narrow() { + if (ASMSIZE == Assembler::int8 || ASMSIZE == Assembler::int16) { + return !(UseZacas && UseZabha); + } + return false; +} + template class CmpxchgTester { // The functions expect arguments to be type represented, not C-ABI argument representation. @@ -126,7 +134,7 @@ class CmpxchgTester { CodeBuffer code(_bb); MacroAssembler _masm(&code); address entry = _masm.pc(); - if (ASMSIZE == Assembler::int8 || ASMSIZE == Assembler::int16) { + if (using_narrow()) { address entry = _masm.pc(); _masm.cmpxchg_narrow_value(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/c_rarg2, ASMSIZE, Assembler::relaxed, Assembler::relaxed, @@ -176,7 +184,7 @@ class CmpxchgTester { } TESTSIZE cmpxchg(intptr_t addr, TESTSIZE expected, TESTSIZE new_value) { - if (ASMSIZE == Assembler::int8 || ASMSIZE == Assembler::int16) { + if (using_narrow()) { return _narrow(addr, expected, new_value, /* dummy result */ 67, -1, -1, -1); } else { return _func(addr, expected, new_value, /* dummy result */ 67); @@ -197,7 +205,7 @@ template static void run_plain_cmpxchg_tests() { TESTSIZE max = std::numeric_limits::max(); TESTSIZE min = std::numeric_limits::min(); - TESTSIZE val[] = {1337, min, max}; + TESTSIZE val[] = {37, min, max}; for (int i = 0; i < 3; i++) { // Normal plain_cmpxchg_test( 0 /* variant */ , val[i] /* start value */, @@ -205,7 +213,7 @@ static void run_plain_cmpxchg_tests() { val[i] /* return */ , 42 /* end value*/, false /* boolean ret*/); plain_cmpxchg_test( 0 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, val[i] /* return */ , val[i] /* end value */, false /* boolean ret*/); plain_cmpxchg_test( 0 /* variant */ , val[i] /* start value */, @@ -213,7 +221,7 @@ static void run_plain_cmpxchg_tests() { 1 /* return */ , 42 /* end value*/, true /* boolean ret*/); plain_cmpxchg_test( 0 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, 0 /* return */ , val[i] /* end value */, true /* boolean ret*/); // result == expected register @@ -222,7 +230,7 @@ static void run_plain_cmpxchg_tests() { val[i] /* return */ , 42 /* end value*/, false /* boolean ret*/); plain_cmpxchg_test( 1 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, val[i] /* return */ , val[i] /* end value */, false /* boolean ret*/); plain_cmpxchg_test( 1 /* variant */ , val[i] /* start value */, @@ -230,7 +238,7 @@ static void run_plain_cmpxchg_tests() { 1 /* return */ , 42 /* end value*/, true /* boolean ret*/); plain_cmpxchg_test( 1 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, 0 /* return */ , val[i] /* end value */, true /* boolean ret*/); // new_value == result register @@ -239,7 +247,7 @@ static void run_plain_cmpxchg_tests() { val[i] /* return */ , 42 /* end value*/, false /* boolean ret*/); plain_cmpxchg_test( 2 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, val[i] /* return */ , val[i] /* end value */, false /* boolean ret*/); plain_cmpxchg_test( 2 /* variant */ , val[i] /* start value */, @@ -247,7 +255,7 @@ static void run_plain_cmpxchg_tests() { 1 /* return */ , 42 /* end value*/, true /* boolean ret*/); plain_cmpxchg_test( 2 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, 0 /* return */ , val[i] /* end value */, true /* boolean ret*/); // expected == new_value register @@ -256,7 +264,7 @@ static void run_plain_cmpxchg_tests() { val[i] /* return */ , val[i] /* end value */, false /* boolean ret*/); plain_cmpxchg_test( 3 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, val[i] /* return */ , val[i] /* end value */, false /* boolean ret*/); plain_cmpxchg_test( 3 /* variant */ , val[i] /* start value */, @@ -264,7 +272,7 @@ static void run_plain_cmpxchg_tests() { 1 /* return */ , val[i] /* end value */, true /* boolean ret*/); plain_cmpxchg_test( 3 /* variant */ , val[i] /* start value */, - 1336 /* expected */, 42 /* new value */, + 36 /* expected */, 42 /* new value */, 0 /* return */ , val[i] /* end value */, true /* boolean ret*/); } } @@ -308,6 +316,18 @@ TEST_VM(RiscV, cmpxchg_uint32_maybe_zacas) { } } +TEST_VM(RiscV, cmpxchg_int16_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { + run_plain_cmpxchg_tests(); + } +} + +TEST_VM(RiscV, cmpxchg_int8_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { + run_plain_cmpxchg_tests(); + } +} + template static void run_narrow_cmpxchg_tests() { CmpxchgTester cmpxchg(0, false); @@ -320,7 +340,6 @@ static void run_narrow_cmpxchg_tests() { TESTSIZE val[] = {121, min, max}; for (int i = 0; i < 3; i++) { for (int j = 0; j < 7; j++) { - // printf("%lu %lX\n", (uint64_t)val[i], (uint64_t)val[i]); memset(data, -1, sizeof(data)); data[i] = val[i]; ret = cmpxchg.cmpxchg((intptr_t)&data[i], val[i], 42); @@ -345,29 +364,35 @@ static void run_narrow_cmpxchg_tests() { } } -TEST_VM(RiscV, cmpxchg_int16_lr_sc) { +TEST_VM(RiscV, cmpxchg_narrow_int16_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_narrow_cmpxchg_tests(); UseZacas = zacas; } -TEST_VM(RiscV, cmpxchg_int16_maybe_zacas) { +TEST_VM(RiscV, cmpxchg_narrow_int16_maybe_zacas) { if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; run_narrow_cmpxchg_tests(); + UseZabha = zabha; } } -TEST_VM(RiscV, cmpxchg_int8_lr_sc) { +TEST_VM(RiscV, cmpxchg_narrow_int8_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_narrow_cmpxchg_tests(); UseZacas = zacas; } -TEST_VM(RiscV, cmpxchg_int8_maybe_zacas) { +TEST_VM(RiscV, cmpxchg_narrow_int8_maybe_zacas) { if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; run_narrow_cmpxchg_tests(); + UseZabha = zabha; } } @@ -486,7 +511,7 @@ TEST_VM(RiscV, cmpxchg_uint32_concurrent_maybe_zacas) { } } -TEST_VM(RiscV, cmpxchg_int16_concurrent_lr_sc) { +TEST_VM(RiscV, cmpxchg_narrow_int16_concurrent_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_concurrent_cmpxchg_tests(); @@ -494,14 +519,17 @@ TEST_VM(RiscV, cmpxchg_int16_concurrent_lr_sc) { UseZacas = zacas; } -TEST_VM(RiscV, cmpxchg_int16_concurrent_maybe_zacas) { +TEST_VM(RiscV, cmpxchg_narrow_int16_concurrent_maybe_zacas) { if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; run_concurrent_cmpxchg_tests(); run_concurrent_alt_cmpxchg_tests(); + UseZabha = zabha; } } -TEST_VM(RiscV, cmpxchg_int8_concurrent_lr_sc) { +TEST_VM(RiscV, cmpxchg_narrow_int8_concurrent_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_concurrent_cmpxchg_tests(); @@ -509,8 +537,25 @@ TEST_VM(RiscV, cmpxchg_int8_concurrent_lr_sc) { UseZacas = zacas; } -TEST_VM(RiscV, cmpxchg_int8_concurrent_maybe_zacas) { +TEST_VM(RiscV, cmpxchg_narrow_int8_concurrent_maybe_zacas) { if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; + run_concurrent_cmpxchg_tests(); + run_concurrent_alt_cmpxchg_tests(); + UseZabha = zabha; + } +} + +TEST_VM(RiscV, cmpxchg_int16_concurrent_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { + run_concurrent_cmpxchg_tests(); + run_concurrent_alt_cmpxchg_tests(); + } +} + +TEST_VM(RiscV, cmpxchg_int8_concurrent_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { run_concurrent_cmpxchg_tests(); run_concurrent_alt_cmpxchg_tests(); } @@ -534,7 +579,7 @@ class WeakCmpxchgTester { _bb = BufferBlob::create("riscvTest", 128); CodeBuffer code(_bb); MacroAssembler _masm(&code); - if (ASMSIZE == Assembler::int8 || ASMSIZE == Assembler::int16) { + if (using_narrow()) { address entry = _masm.pc(); _masm.weak_cmpxchg_narrow_value(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/ c_rarg2, ASMSIZE, Assembler::relaxed, Assembler::relaxed, @@ -554,7 +599,7 @@ class WeakCmpxchgTester { } TESTSIZE weak_cmpxchg(intptr_t addr, TESTSIZE expected, TESTSIZE new_value) { - if (ASMSIZE == Assembler::int8 || ASMSIZE == Assembler::int16) { + if (using_narrow()) { return _narrow_weak(addr, expected, new_value, /* dummy result */ 67, -1, -1, -1); } else { return _weak(addr, expected, new_value, /* dummy result */ 67); @@ -624,28 +669,46 @@ TEST_VM(RiscV, weak_cmpxchg_uint32_maybe_zacas) { } } -TEST_VM(RiscV, weak_cmpxchg_int16_lr_sc) { +TEST_VM(RiscV, weak_cmpxchg_narrow_int16_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_weak_cmpxchg_tests(); UseZacas = zacas; } -TEST_VM(RiscV, weak_cmpxchg_int8_lr_sc) { +TEST_VM(RiscV, weak_cmpxchg_narrow_int8_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_weak_cmpxchg_tests(); UseZacas = zacas; } -TEST_VM(RiscV, weak_cmpxchg_int16_maybe_zacas) { +TEST_VM(RiscV, weak_cmpxchg_narrow_int16_maybe_zacas) { if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; + run_weak_cmpxchg_tests(); + UseZabha = zabha; + } +} + +TEST_VM(RiscV, weak_cmpxchg_narrow_int8_maybe_zacas) { + if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; + run_weak_cmpxchg_tests(); + UseZabha = zabha; + } +} + +TEST_VM(RiscV, weak_cmpxchg_int16_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { run_weak_cmpxchg_tests(); } } -TEST_VM(RiscV, weak_cmpxchg_int8_maybe_zacas) { - if (UseZacas) { +TEST_VM(RiscV, weak_cmpxchg_int8_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { run_weak_cmpxchg_tests(); } } @@ -724,7 +787,7 @@ TEST_VM(RiscV, weak_cmpxchg_int32_concurrent_maybe_zacas) { } } -TEST_VM(RiscV, weak_cmpxchg_int16_concurrent_lr_sc) { +TEST_VM(RiscV, weak_cmpxchg_narrow_int16_concurrent_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_concurrent_weak_cmpxchg_tests(); @@ -732,14 +795,17 @@ TEST_VM(RiscV, weak_cmpxchg_int16_concurrent_lr_sc) { UseZacas = zacas; } -TEST_VM(RiscV, weak_cmpxchg_int16_concurrent_maybe_zacas) { +TEST_VM(RiscV, weak_cmpxchg_narrow_int16_concurrent_maybe_zacas) { if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; run_concurrent_weak_cmpxchg_tests(); run_concurrent_alt_weak_cmpxchg_tests(); + UseZabha = zabha; } } -TEST_VM(RiscV, weak_cmpxchg_int8_concurrent_lr_sc) { +TEST_VM(RiscV, weak_cmpxchg_narrow_int8_concurrent_lr_sc) { bool zacas = UseZacas; UseZacas = false; run_concurrent_weak_cmpxchg_tests(); @@ -747,8 +813,25 @@ TEST_VM(RiscV, weak_cmpxchg_int8_concurrent_lr_sc) { UseZacas = zacas; } -TEST_VM(RiscV, weak_cmpxchg_int8_concurrent_maybe_zacas) { +TEST_VM(RiscV, weak_cmpxchg_narrow_int8_concurrent_maybe_zacas) { if (UseZacas) { + bool zabha = UseZabha; + UseZabha = false; + run_concurrent_weak_cmpxchg_tests(); + run_concurrent_alt_weak_cmpxchg_tests(); + UseZabha = zabha; + } +} + +TEST_VM(RiscV, weak_cmpxchg_int16_concurrent_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { + run_concurrent_weak_cmpxchg_tests(); + run_concurrent_alt_weak_cmpxchg_tests(); + } +} + +TEST_VM(RiscV, weak_cmpxchg_int8_concurrent_maybe_zacas_zabha) { + if (UseZacas && UseZabha) { run_concurrent_weak_cmpxchg_tests(); run_concurrent_alt_weak_cmpxchg_tests(); }