openjdk/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad

//
// Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//

source_hpp %{

#include "gc/shared/gc_globals.hpp"
#include "gc/z/c2/zBarrierSetC2.hpp"
#include "gc/z/zThreadLocalData.hpp"

%}

source %{

#include "gc/z/zBarrierSetAssembler.hpp"

static void z_color(MacroAssembler* masm, const MachNode* node, Register dst, Register src) {
  assert_different_registers(src, dst);
  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
  __ movzw(dst, barrier_Relocation::unpatched);
  __ orr(dst, dst, src, Assembler::LSL, ZPointerLoadShift);
}

static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
  __ lsr(ref, ref, ZPointerLoadShift);
}

static void z_keep_alive_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
  __ movzw(tmp, barrier_Relocation::unpatched);
  __ tst(ref, tmp);
  ZLoadBarrierStubC2Aarch64* const stub = ZLoadBarrierStubC2Aarch64::create(node, ref_addr, ref);
  __ br(Assembler::NE, *stub->entry());
  z_uncolor(masm, node, ref);
  __ bind(*stub->continuation());
}

static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
  const bool on_non_strong =
      ((node->barrier_data() & ZBarrierWeak) != 0) ||
      ((node->barrier_data() & ZBarrierPhantom) != 0);

  if (on_non_strong) {
    z_keep_alive_load_barrier(masm, node, ref_addr, ref, tmp);
    return;
  }

  if (node->barrier_data() == ZBarrierElided) {
    z_uncolor(masm, node, ref);
    return;
  }

  ZLoadBarrierStubC2Aarch64* const stub = ZLoadBarrierStubC2Aarch64::create(node, ref_addr, ref, __ offset());
  if (stub->is_test_and_branch_reachable()) {
    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
    __ tbnz(ref, barrier_Relocation::unpatched, *stub->entry());
  } else {
    Label good;
    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
    __ tbz(ref, barrier_Relocation::unpatched, good);
    __ b(*stub->entry());
    __ bind(good);
  }
  z_uncolor(masm, node, ref);
  __ bind(*stub->continuation());
}

static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
  if (node->barrier_data() == ZBarrierElided) {
    z_color(masm, node, rnew_zpointer, rnew_zaddress);
  } else {
    bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
    bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
    ZStoreBarrierStubC2Aarch64* const stub = ZStoreBarrierStubC2Aarch64::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic, is_nokeepalive);
    ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
    bs_asm->store_barrier_fast(masm, ref_addr, rnew_zaddress, rnew_zpointer, tmp, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
  }
}

%}

// Load Pointer
instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
%{
  match(Set dst (LoadP mem));
  predicate(UseZGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
  effect(TEMP dst, KILL cr);
  // The main load is a candidate to implement implicit null checks, as long as
  // legitimize_address() does not require a preceding lea instruction to
  // materialize the memory operand. The absence of a preceding lea instruction
  // is guaranteed for immLoffset8 memory operands, because these do not lead to
  // out-of-range offsets (see definition of immLoffset8). Fortunately,
  // immLoffset8 memory operands are the most common ones in practice.
  ins_is_late_expanded_null_check_candidate(opnd_array(1)->opcode() == INDOFFL8);

  ins_cost(4 * INSN_COST);

  format %{ "ldr  $dst, $mem" %}

  ins_encode %{
    Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    if (ref_addr.getMode() == Address::base_plus_offset) {
      // Fix up any out-of-range offsets.
      assert_different_registers(rscratch2, as_Register($mem$$base));
      assert_different_registers(rscratch2, $dst$$Register);
      int size = 8;
      assert(!this->is_late_expanded_null_check_candidate() ||
             !MacroAssembler::legitimize_address_requires_lea(ref_addr, size),
             "an instruction that can be used for implicit null checking should emit the candidate memory access first");
      ref_addr = __ legitimize_address(ref_addr, size, rscratch2);
    }
    __ ldr($dst$$Register, ref_addr);
    z_load_barrier(masm, this, ref_addr, $dst$$Register, rscratch1);
  %}

  ins_pipe(iload_reg_mem);
%}

// Load Pointer Volatile
instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
%{
  match(Set dst (LoadP mem));
  predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
  effect(TEMP dst, KILL cr);

  ins_cost(VOLATILE_REF_COST);

  format %{ "ldar  $dst, $mem\t" %}

  ins_encode %{
    const Address ref_addr = Address($mem$$Register);
    __ ldar($dst$$Register, $mem$$Register);
    z_load_barrier(masm, this, ref_addr, $dst$$Register, rscratch1);
  %}

  ins_pipe(pipe_serial);
%}

// Store Pointer
instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
%{
  predicate(UseZGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
  match(Set mem (StoreP mem src));
  effect(TEMP tmp, KILL cr);

  ins_cost(125); // XXX
  format %{ "movq    $mem, $src\t# ptr" %}
  ins_encode %{
    const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    z_store_barrier(masm, this, ref_addr, $src$$Register, $tmp$$Register, rscratch2, false /* is_atomic */);
    __ str($tmp$$Register, ref_addr);
  %}
  ins_pipe(pipe_serial);
%}

// Store Pointer Volatile
instruct zStorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
%{
  predicate(UseZGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
  match(Set mem (StoreP mem src));
  effect(TEMP tmp, KILL cr);

  ins_cost(125); // XXX
  format %{ "movq    $mem, $src\t# ptr" %}
  ins_encode %{
    const Address ref_addr = Address($mem$$Register);
    z_store_barrier(masm, this, ref_addr, $src$$Register, $tmp$$Register, rscratch2, false /* is_atomic */);
    __ stlr($tmp$$Register, $mem$$Register);
  %}
  ins_pipe(pipe_serial);
%}

instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);

  ins_cost(2 * VOLATILE_REF_COST);

  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
            "cset    $res, EQ" %}

  ins_encode %{
    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
    Address ref_addr($mem$$Register);
    z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
               false /* acquire */, true /* release */, false /* weak */, noreg);
    __ cset($res$$Register, Assembler::EQ);
  %}

  ins_pipe(pipe_slow);
%}

instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
  predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);

  ins_cost(2 * VOLATILE_REF_COST);

  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
            "cset    $res, EQ" %}

  ins_encode %{
    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
    Address ref_addr($mem$$Register);
    z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
               true /* acquire */, true /* release */, false /* weak */, noreg);
    __ cset($res$$Register, Assembler::EQ);
  %}

  ins_pipe(pipe_slow);
%}


instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);

  ins_cost(2 * VOLATILE_REF_COST);

  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
            "cset    $res, EQ" %}

  ins_encode %{
    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
    Address ref_addr($mem$$Register);
    z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
               false /* acquire */, true /* release */, false /* weak */, $res$$Register);
    z_uncolor(masm, this, $res$$Register);
  %}

  ins_pipe(pipe_slow);
%}

instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
  predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);

  ins_cost(2 * VOLATILE_REF_COST);

  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
            "cset    $res, EQ" %}

  ins_encode %{
    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
    Address ref_addr($mem$$Register);
    z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
               true /* acquire */, true /* release */, false /* weak */, $res$$Register);
    z_uncolor(masm, this, $res$$Register);
  %}

  ins_pipe(pipe_slow);
%}

instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
  match(Set prev (GetAndSetP mem newv));
  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
  effect(TEMP prev, KILL cr);

  ins_cost(2 * VOLATILE_REF_COST);

  format %{ "atomic_xchg  $prev, $newv, [$mem]" %}

  ins_encode %{
    z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
    __ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
    z_uncolor(masm, this, $prev$$Register);
  %}

  ins_pipe(pipe_serial);
%}

instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
  match(Set prev (GetAndSetP mem newv));
  predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
  effect(TEMP prev, KILL cr);

  ins_cost(2 * VOLATILE_REF_COST);

  format %{ "atomic_xchg  $prev, $newv, [$mem]" %}

  ins_encode %{
    z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
    __ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
    z_uncolor(masm, this, $prev$$Register);
  %}

  ins_pipe(pipe_serial);
%}