8274615: Support relaxed atomic add for linux-aarch64

Reviewed-by: aph, dholmes

Author: Kim Barrett

src/hotspot/cpu/aarch64/atomic_aarch64.hpp

@@ -37,6 +37,8 @@ typedef uint64_t (*aarch64_atomic_stub_t)(volatile void *ptr, uint64_t arg1, uin
 // Pointers to stubs
 extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_4_impl;
 extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_8_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_4_relaxed_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_8_relaxed_impl;
 extern aarch64_atomic_stub_t aarch64_atomic_xchg_4_impl;
 extern aarch64_atomic_stub_t aarch64_atomic_xchg_8_impl;
 extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_1_impl;

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -6209,10 +6209,16 @@ class StubGenerator: public StubCodeGenerator {
     __ ret(lr);
   }
 
-  void gen_ldaddal_entry(Assembler::operand_size size) {
+  void gen_ldadd_entry(Assembler::operand_size size, atomic_memory_order order) {
     Register prev = r2, addr = c_rarg0, incr = c_rarg1;
-    __ ldaddal(size, incr, prev, addr);
-    __ membar(Assembler::StoreStore|Assembler::StoreLoad);
+    // If not relaxed, then default to conservative.  Relaxed is the only
+    // case we use enough to be worth specializing.
+    if (order == memory_order_relaxed) {
+      __ ldadd(size, incr, prev, addr);
+    } else {
+      __ ldaddal(size, incr, prev, addr);
+      __ membar(Assembler::StoreStore|Assembler::StoreLoad);
+    }
     if (size == Assembler::xword) {
       __ mov(r0, prev);
     } else {
@@ -6242,12 +6248,21 @@ class StubGenerator: public StubCodeGenerator {
     StubCodeMark mark(this, "StubRoutines", "atomic entry points");
     address first_entry = __ pc();
 
-    // All memory_order_conservative
+    // ADD, memory_order_conservative
     AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
-    gen_ldaddal_entry(Assembler::word);
+    gen_ldadd_entry(Assembler::word, memory_order_conservative);
     AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
-    gen_ldaddal_entry(Assembler::xword);
+    gen_ldadd_entry(Assembler::xword, memory_order_conservative);
+
+    // ADD, memory_order_relaxed
+    AtomicStubMark mark_fetch_add_4_relaxed
+      (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl);
+    gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed);
+    AtomicStubMark mark_fetch_add_8_relaxed
+      (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl);
+    gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed);
 
+    // XCHG, memory_order_conservative
     AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
     gen_swpal_entry(Assembler::word);
     AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl);
@@ -7447,6 +7462,8 @@ void StubGenerator_generate(CodeBuffer* code, bool all) {
 
 DEFAULT_ATOMIC_OP(fetch_add, 4, )
 DEFAULT_ATOMIC_OP(fetch_add, 8, )
+DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
+DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
 DEFAULT_ATOMIC_OP(xchg, 4, )
 DEFAULT_ATOMIC_OP(xchg, 8, )
 DEFAULT_ATOMIC_OP(cmpxchg, 1, )

src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S

@@ -47,6 +47,28 @@ aarch64_atomic_fetch_add_4_default_impl:
         mov     w0, w2
         ret
 
+        .global aarch64_atomic_fetch_add_8_relaxed_default_impl
+        .align 5
+aarch64_atomic_fetch_add_8_relaxed_default_impl:
+        prfm    pstl1strm, [x0]
+0:      ldxr    x2, [x0]
+        add     x8, x2, x1
+        stxr    w9, x8, [x0]
+        cbnz    w9, 0b
+        mov     x0, x2
+        ret
+
+        .global aarch64_atomic_fetch_add_4_relaxed_default_impl
+        .align 5
+aarch64_atomic_fetch_add_4_relaxed_default_impl:
+        prfm    pstl1strm, [x0]
+0:      ldxr    w2, [x0]
+        add     w8, w2, w1
+        stxr    w9, w8, [x0]
+        cbnz    w9, 0b
+        mov     w0, w2
+        ret
+
         .globl aarch64_atomic_xchg_4_default_impl
         .align 5
 aarch64_atomic_xchg_4_default_impl:

src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -87,9 +87,14 @@ inline D Atomic::PlatformAdd<4>::fetch_and_add(D volatile* dest, I add_value,
                                                atomic_memory_order order) const {
   STATIC_ASSERT(4 == sizeof(I));
   STATIC_ASSERT(4 == sizeof(D));
-  D old_value
-    = atomic_fastcall(aarch64_atomic_fetch_add_4_impl, dest, add_value);
-  return old_value;
+  aarch64_atomic_stub_t stub;
+  switch (order) {
+  case memory_order_relaxed:
+    stub = aarch64_atomic_fetch_add_4_relaxed_impl; break;
+  default:
+    stub = aarch64_atomic_fetch_add_4_impl; break;
+  }
+  return atomic_fastcall(stub, dest, add_value);
 }
 
 template<>
@@ -98,9 +103,14 @@ inline D Atomic::PlatformAdd<8>::fetch_and_add(D volatile* dest, I add_value,
                                                atomic_memory_order order) const {
   STATIC_ASSERT(8 == sizeof(I));
   STATIC_ASSERT(8 == sizeof(D));
-  D old_value
-    = atomic_fastcall(aarch64_atomic_fetch_add_8_impl, dest, add_value);
-  return old_value;
+  aarch64_atomic_stub_t stub;
+  switch (order) {
+  case memory_order_relaxed:
+    stub = aarch64_atomic_fetch_add_8_relaxed_impl; break;
+  default:
+    stub = aarch64_atomic_fetch_add_8_impl; break;
+  }
+  return atomic_fastcall(stub, dest, add_value);
 }
 
 template<>