8331418: ZGC: generalize barrier liveness logic

Reviewed-by: mdoerr, aboldtch, fyang, eosterlund

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp

@@ -476,7 +476,7 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
   GrowableArray<RegisterData> registers;
   VMReg prev_vm_reg = VMRegImpl::Bad();
 
-  RegMaskIterator rmi(stub->live());
+  RegMaskIterator rmi(stub->preserve_set());
   while (rmi.has_next()) {
     OptoReg::Name opto_reg = rmi.next();
     VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
@@ -491,7 +491,7 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
         index = registers.append(reg_data);
       }
     } else if (vm_reg->is_FloatRegister()) {
-      // We have size encoding in OptoReg of stub->live()
+      // We have size encoding in OptoReg of stub->preserve_set()
       // After encoding, float/neon/sve register has only one slot in regmask
       // Decode it to get the actual size
       VMReg vm_reg_base = vm_reg->as_FloatRegister()->as_VMReg();
@@ -532,12 +532,8 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
     }
   }
 
-  // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
+  // Remove C-ABI SOE registers and scratch regs
-  if (stub->result() != noreg) {
+  _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9);
-    _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->result());
-  } else {
-    _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9);
-  }
 
   // Remove C-ABI SOE fp registers
   _fp_regs -= FloatRegSet::range(v8, v15);

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp

@@ -282,7 +282,7 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
 #define __ _masm->
 
 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler *masm, BarrierStubC2 *stub)
-  : _masm(masm), _reg_mask(stub->live()), _result_reg(stub->result()) {
+  : _masm(masm), _reg_mask(stub->preserve_set()) {
 
   const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord;
   _frame_size = align_up(register_save_size, frame::alignment_in_bytes)
@@ -317,11 +317,6 @@ int SaveLiveRegisters::iterate_over_register_mask(IterationAction action, int of
     if (vm_reg->is_Register()) {
       Register std_reg = vm_reg->as_Register();
 
-      // '_result_reg' will hold the end result of the operation. Its content must thus not be preserved.
-      if (std_reg == _result_reg) {
-        continue;
-      }
-
       if (std_reg->encoding() >= R2->encoding() && std_reg->encoding() <= R12->encoding()) {
         reg_save_index++;
 

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp

@@ -98,7 +98,6 @@ public:
 class SaveLiveRegisters {
   MacroAssembler* _masm;
   RegMask _reg_mask;
-  Register _result_reg;
   int _frame_size;
 
  public:

src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp

@@ -396,7 +396,7 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
 
 void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
   // Record registers that needs to be saved/restored
-  RegMaskIterator rmi(stub->live());
+  RegMaskIterator rmi(stub->preserve_set());
   while (rmi.has_next()) {
     const OptoReg::Name opto_reg = rmi.next();
     if (OptoReg::is_reg(opto_reg)) {
@@ -414,12 +414,8 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
     }
   }
 
-  // Remove C-ABI SOE registers, tmp regs and _ref register that will be updated
+  // Remove C-ABI SOE registers and tmp regs
-  if (stub->result() != noreg) {
+  _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2, x5) + RegSet::of(x8, x9);
-    _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2) + RegSet::of(x8, x9) + RegSet::of(x5, stub->result());
-  } else {
-    _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2, x5) + RegSet::of(x8, x9);
-  }
 }
 
 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)

src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp

@@ -613,19 +613,12 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
   caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
   caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
 
-  if (stub->result() != noreg) {
-    caller_saved.Remove(OptoReg::as_OptoReg(stub->result()->as_VMReg()));
-  }
-
-  // Create mask of live registers
-  RegMask live = stub->live();
-
   int gp_spill_size = 0;
   int opmask_spill_size = 0;
   int xmm_spill_size = 0;
 
   // Record registers that needs to be saved/restored
-  RegMaskIterator rmi(live);
+  RegMaskIterator rmi(stub->preserve_set());
   while (rmi.has_next()) {
     const OptoReg::Name opto_reg = rmi.next();
     const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -87,15 +87,16 @@ static BarrierSetC2State* barrier_set_state() {
   return reinterpret_cast<BarrierSetC2State*>(Compile::current()->barrier_set_state());
 }
 
-BarrierStubC2::BarrierStubC2(const MachNode* node)
-  : _node(node),
-    _entry(),
-    _continuation() {}
-
 RegMask& BarrierStubC2::live() const {
   return *barrier_set_state()->live(_node);
 }
 
+BarrierStubC2::BarrierStubC2(const MachNode* node)
+  : _node(node),
+    _entry(),
+    _continuation(),
+    _preserve(live()) {}
+
 Label* BarrierStubC2::entry() {
   // The _entry will never be bound when in_scratch_emit_size() is true.
   // However, we still need to return a label that is not bound now, but
@@ -108,6 +109,27 @@ Label* BarrierStubC2::continuation() {
   return &_continuation;
 }
 
+void BarrierStubC2::preserve(Register r) {
+  const VMReg vm_reg = r->as_VMReg();
+  assert(vm_reg->is_Register(), "r must be a general-purpose register");
+  _preserve.Insert(OptoReg::as_OptoReg(vm_reg));
+}
+
+void BarrierStubC2::dont_preserve(Register r) {
+  VMReg vm_reg = r->as_VMReg();
+  assert(vm_reg->is_Register(), "r must be a general-purpose register");
+  // Subtract the given register and all its sub-registers (e.g. {R11, R11_H}
+  // for r11 in aarch64).
+  do {
+    _preserve.Remove(OptoReg::as_OptoReg(vm_reg));
+    vm_reg = vm_reg->next();
+  } while (vm_reg->is_Register() && !vm_reg->is_concrete());
+}
+
+const RegMask& BarrierStubC2::preserve_set() const {
+  return _preserve;
+}
+
 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
   DecoratorSet decorators = access.decorators();
 
@@ -828,6 +850,7 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
   PhaseRegAlloc* const regalloc = C->regalloc();
   RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
   BarrierSetAssembler* const bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  BarrierSetC2State* bs_state = barrier_set_state();
   Block_List worklist;
 
   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
@@ -850,6 +873,14 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
     for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
       const Node* const node = block->get_node(i);
 
+      // If this node tracks out-liveness, update it
+      if (!bs_state->needs_livein_data()) {
+        RegMask* const regs = bs_state->live(node);
+        if (regs != nullptr) {
+          regs->OR(new_live);
+        }
+      }
+
       // Remove def bits
       const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
       const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
@@ -873,10 +904,12 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
         }
       }
 
-       // If this node tracks liveness, update it
+      // If this node tracks in-liveness, update it
-      RegMask* const regs = barrier_set_state()->live(node);
+      if (bs_state->needs_livein_data()) {
-      if (regs != NULL) {
+        RegMask* const regs = bs_state->live(node);
-        regs->OR(new_live);
+        if (regs != nullptr) {
+          regs->OR(new_live);
+        }
       }
     }
 

src/hotspot/share/gc/shared/c2/barrierSetC2.hpp

@@ -227,6 +227,7 @@ public:
   }
 
   virtual bool needs_liveness_data(const MachNode* mach) const = 0;
+  virtual bool needs_livein_data() const = 0;
 };
 
 // This class represents the slow path in a C2 barrier. It is defined by a
@@ -238,14 +239,28 @@ protected:
   const MachNode* _node;
   Label           _entry;
   Label           _continuation;
+  RegMask         _preserve;
+
+  // Registers that are live-in/live-out of the entire memory access
+  // implementation (possibly including multiple barriers). Whether live-in or
+  // live-out registers are returned depends on
+  // BarrierSetC2State::needs_livein_data().
+  RegMask& live() const;
 
 public:
   BarrierStubC2(const MachNode* node);
-  RegMask& live() const;
+
+  // Entry point to the stub.
   Label* entry();
+  // Return point from the stub (typically end of barrier).
   Label* continuation();
 
-  virtual Register result() const = 0;
+  // Preserve the value in reg across runtime calls in this barrier.
+  void preserve(Register reg);
+  // Do not preserve the value in reg across runtime calls in this barrier.
+  void dont_preserve(Register reg);
+  // Set of registers whose value needs to be preserved across runtime calls in this barrier.
+  const RegMask& preserve_set() const;
 };
 
 // This is the top-level class for the backend of the Access API in C2.

src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp

@@ -142,6 +142,10 @@ public:
     return mach->barrier_data() != ZBarrierElided;
   }
 
+  bool needs_livein_data() const {
+    return true;
+  }
+
   void inc_trampoline_stubs_count() {
     assert(_trampoline_stubs_count != INT_MAX, "Overflow");
     ++_trampoline_stubs_count;
@@ -200,6 +204,9 @@ ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, R
     _ref(ref) {
   assert_different_registers(ref, ref_addr.base());
   assert_different_registers(ref, ref_addr.index());
+  // The runtime call updates the value of ref, so we should not spill and
+  // reload its outdated value.
+  dont_preserve(ref);
 }
 
 Address ZLoadBarrierStubC2::ref_addr() const {
@@ -210,10 +217,6 @@ Register ZLoadBarrierStubC2::ref() const {
   return _ref;
 }
 
-Register ZLoadBarrierStubC2::result() const {
-  return ref();
-}
-
 address ZLoadBarrierStubC2::slow_path() const {
   const uint8_t barrier_data = _node->barrier_data();
   DecoratorSet decorators = DECORATORS_NONE;
@@ -272,10 +275,6 @@ bool ZStoreBarrierStubC2::is_atomic() const {
   return _is_atomic;
 }
 
-Register ZStoreBarrierStubC2::result() const {
-  return noreg;
-}
-
 void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
   ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
 }

src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp

@@ -51,7 +51,6 @@ static int stubs_start_offset();
   ZBarrierStubC2(const MachNode* node);
 
 public:
-  virtual Register result() const = 0;
   virtual void emit_code(MacroAssembler& masm) = 0;
 };
 
@@ -70,7 +69,6 @@ public:
   Register ref() const;
   address slow_path() const;
 
-  virtual Register result() const;
   virtual void emit_code(MacroAssembler& masm);
 };
 
@@ -94,7 +92,6 @@ public:
   bool is_native() const;
   bool is_atomic() const;
 
-  virtual Register result() const;
   virtual void emit_code(MacroAssembler& masm);
 };