8246373: AArch64: Refactor register spilling code in ZGC barriers

Tidy up code spilling registers, reduce in some cases.

Reviewed-by: aph, eosterlund

Author: Stuart Monteith

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -67,8 +67,6 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
   assert_different_registers(rscratch1, rscratch2, src.base());
   assert_different_registers(rscratch1, rscratch2, dst);
 
-  RegSet savedRegs = RegSet::range(r0, r28) - RegSet::of(dst, rscratch1, rscratch2);
-
   Label done;
 
   // Load bad mask into scratch register.
@@ -82,37 +80,21 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
 
   __ enter();
 
-  __ push(savedRegs, sp);
+  __ push_call_clobbered_registers_except(RegSet::of(dst));
 
   if (c_rarg0 != dst) {
     __ mov(c_rarg0, dst);
   }
   __ mov(c_rarg1, rscratch2);
 
-  int step = 4 * wordSize;
-  __ mov(rscratch2, -step);
-  __ sub(sp, sp, step);
-
-  for (int i = 28; i >= 4; i -= 4) {
-    __ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
-        as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch2)));
-  }
-  __ st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2),
-      as_FloatRegister(3), __ T1D, Address(sp));
-
   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
 
-  for (int i = 0; i <= 28; i += 4) {
-    __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
-        as_FloatRegister(i+3), __ T1D, Address(__ post(sp, step)));
-  }
-
   // Make sure dst has the return value.
   if (dst != r0) {
     __ mov(dst, r0);
   }
 
-  __ pop(savedRegs, sp);
+  __ pop_call_clobbered_registers_except(RegSet::of(dst));
   __ leave();
 
   __ bind(done);
@@ -170,7 +152,7 @@ void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
 
   assert_different_registers(src, count, rscratch1);
 
-  __ pusha();
+  __ push(saved_regs, sp);
 
   if (count == c_rarg0) {
     if (src == c_rarg1) {
@@ -189,7 +171,8 @@ void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
 
   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
 
-  __ popa();
+  __ pop(saved_regs, sp);
+
   BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
 }
 
@@ -295,25 +278,15 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
                                                                  DecoratorSet decorators) const {
   __ prologue("zgc_load_barrier stub", false);
 
-  // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0.
-  for (int i = 0; i < 32; i += 2) {
-    __ stpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ pre(sp,-16)));
-  }
-
-  const RegSet save_regs = RegSet::range(r1, r28);
-  __ push(save_regs, sp);
+  __ push_call_clobbered_registers_except(RegSet::of(r0));
 
   // Setup arguments
   __ load_parameter(0, c_rarg0);
   __ load_parameter(1, c_rarg1);
 
   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
 
-  __ pop(save_regs, sp);
-
-  for (int i = 30; i >= 0; i -= 2) {
-    __ ldpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ post(sp, 16)));
-  }
+  __ pop_call_clobbered_registers_except(RegSet::of(r0));
 
   __ epilogue();
 }

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -2626,9 +2626,9 @@ void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
   fatal("DEBUG MESSAGE: %s", msg);
 }
 
-void MacroAssembler::push_call_clobbered_registers() {
+void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) {
   int step = 4 * wordSize;
-  push(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
+  push(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
   sub(sp, sp, step);
   mov(rscratch1, -step);
   // Push v0-v7, v16-v31.
@@ -2641,14 +2641,14 @@ void MacroAssembler::push_call_clobbered_registers() {
       as_FloatRegister(3), T1D, Address(sp));
 }
 
-void MacroAssembler::pop_call_clobbered_registers() {
+void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) {
   for (int i = 0; i < 32; i += 4) {
     if (i <= v7->encoding() || i >= v16->encoding())
       ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
           as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize)));
   }
 
-  pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
+  pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
 }
 
 void MacroAssembler::push_CPU_state(bool save_vectors) {

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -477,9 +477,18 @@ class MacroAssembler: public Assembler {
   // Push and pop everything that might be clobbered by a native
   // runtime call except rscratch1 and rscratch2.  (They are always
   // scratch, so we don't have to protect them.)  Only save the lower
-  // 64 bits of each vector register.
-  void push_call_clobbered_registers();
-  void pop_call_clobbered_registers();
+  // 64 bits of each vector register. Additonal registers can be excluded
+  // in a passed RegSet.
+  void push_call_clobbered_registers_except(RegSet exclude);
+  void pop_call_clobbered_registers_except(RegSet exclude);
+
+  void push_call_clobbered_registers() {
+    push_call_clobbered_registers_except(RegSet());
+  }
+  void pop_call_clobbered_registers() {
+    pop_call_clobbered_registers_except(RegSet());
+  }
+
 
   // now mov instructions for loading absolute addresses and 32 or
   // 64 bit integers