Home | History | Annotate | Download | only in base 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 // This file is an internal atomic implementation, use base/atomicops.h instead.
      6 
      7 // TODO(rmcilroy): Investigate whether we can use __sync__ intrinsics instead of
      8 //                 the hand coded assembly without introducing perf regressions.
      9 // TODO(rmcilroy): Investigate whether we can use acquire / release versions of
     10 //                 exclusive load / store assembly instructions and do away with
     11 //                 the barriers.
     12 
     13 #ifndef BASE_ATOMICOPS_INTERNALS_ARM64_GCC_H_
     14 #define BASE_ATOMICOPS_INTERNALS_ARM64_GCC_H_
     15 
     16 #if defined(OS_QNX)
     17 #include <sys/cpuinline.h>
     18 #endif
     19 
     20 namespace base {
     21 namespace subtle {
     22 
     23 inline void MemoryBarrier() {
     24   __asm__ __volatile__ ("dmb ish" ::: "memory");  // NOLINT
     25 }
     26 
     27 // NoBarrier versions of the operation include "memory" in the clobber list.
     28 // This is not required for direct usage of the NoBarrier versions of the
     29 // operations. However this is required for correctness when they are used as
     30 // part of the Acquire or Release versions, to ensure that nothing from outside
     31 // the call is reordered between the operation and the memory barrier. This does
     32 // not change the code generated, so has no or minimal impact on the
     33 // NoBarrier operations.
     34 
     35 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr,
     36                                          Atomic32 old_value,
     37                                          Atomic32 new_value) {
     38   Atomic32 prev;
     39   int32_t temp;
     40 
     41   __asm__ __volatile__ (  // NOLINT
     42     "0:                                    \n\t"
     43     "ldxr %w[prev], %[ptr]                 \n\t"  // Load the previous value.
     44     "cmp %w[prev], %w[old_value]           \n\t"
     45     "bne 1f                                \n\t"
     46     "stxr %w[temp], %w[new_value], %[ptr]  \n\t"  // Try to store the new value.
     47     "cbnz %w[temp], 0b                     \n\t"  // Retry if it did not work.
     48     "1:                                    \n\t"
     49     : [prev]"=&r" (prev),
     50       [temp]"=&r" (temp),
     51       [ptr]"+Q" (*ptr)
     52     : [old_value]"IJr" (old_value),
     53       [new_value]"r" (new_value)
     54     : "cc", "memory"
     55   );  // NOLINT
     56 
     57   return prev;
     58 }
     59 
     60 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr,
     61                                          Atomic32 new_value) {
     62   Atomic32 result;
     63   int32_t temp;
     64 
     65   __asm__ __volatile__ (  // NOLINT
     66     "0:                                    \n\t"
     67     "ldxr %w[result], %[ptr]               \n\t"  // Load the previous value.
     68     "stxr %w[temp], %w[new_value], %[ptr]  \n\t"  // Try to store the new value.
     69     "cbnz %w[temp], 0b                     \n\t"  // Retry if it did not work.
     70     : [result]"=&r" (result),
     71       [temp]"=&r" (temp),
     72       [ptr]"+Q" (*ptr)
     73     : [new_value]"r" (new_value)
     74     : "memory"
     75   );  // NOLINT
     76 
     77   return result;
     78 }
     79 
     80 inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr,
     81                                           Atomic32 increment) {
     82   Atomic32 result;
     83   int32_t temp;
     84 
     85   __asm__ __volatile__ (  // NOLINT
     86     "0:                                       \n\t"
     87     "ldxr %w[result], %[ptr]                  \n\t"  // Load the previous value.
     88     "add %w[result], %w[result], %w[increment]\n\t"
     89     "stxr %w[temp], %w[result], %[ptr]        \n\t"  // Try to store the result.
     90     "cbnz %w[temp], 0b                        \n\t"  // Retry on failure.
     91     : [result]"=&r" (result),
     92       [temp]"=&r" (temp),
     93       [ptr]"+Q" (*ptr)
     94     : [increment]"IJr" (increment)
     95     : "memory"
     96   );  // NOLINT
     97 
     98   return result;
     99 }
    100 
    101 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr,
    102                                         Atomic32 increment) {
    103   MemoryBarrier();
    104   Atomic32 result = NoBarrier_AtomicIncrement(ptr, increment);
    105   MemoryBarrier();
    106 
    107   return result;
    108 }
    109 
    110 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr,
    111                                        Atomic32 old_value,
    112                                        Atomic32 new_value) {
    113   Atomic32 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
    114   MemoryBarrier();
    115 
    116   return prev;
    117 }
    118 
    119 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr,
    120                                        Atomic32 old_value,
    121                                        Atomic32 new_value) {
    122   MemoryBarrier();
    123   Atomic32 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
    124 
    125   return prev;
    126 }
    127 
    128 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) {
    129   *ptr = value;
    130 }
    131 
    132 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) {
    133   *ptr = value;
    134   MemoryBarrier();
    135 }
    136 
    137 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) {
    138   __asm__ __volatile__ (  // NOLINT
    139     "stlr %w[value], %[ptr]  \n\t"
    140     : [ptr]"=Q" (*ptr)
    141     : [value]"r" (value)
    142     : "memory"
    143   );  // NOLINT
    144 }
    145 
    146 inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) {
    147   return *ptr;
    148 }
    149 
    150 inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) {
    151   Atomic32 value;
    152 
    153   __asm__ __volatile__ (  // NOLINT
    154     "ldar %w[value], %[ptr]  \n\t"
    155     : [value]"=r" (value)
    156     : [ptr]"Q" (*ptr)
    157     : "memory"
    158   );  // NOLINT
    159 
    160   return value;
    161 }
    162 
    163 inline Atomic32 Release_Load(volatile const Atomic32* ptr) {
    164   MemoryBarrier();
    165   return *ptr;
    166 }
    167 
    168 // 64-bit versions of the operations.
    169 // See the 32-bit versions for comments.
    170 
    171 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr,
    172                                          Atomic64 old_value,
    173                                          Atomic64 new_value) {
    174   Atomic64 prev;
    175   int32_t temp;
    176 
    177   __asm__ __volatile__ (  // NOLINT
    178     "0:                                    \n\t"
    179     "ldxr %[prev], %[ptr]                  \n\t"
    180     "cmp %[prev], %[old_value]             \n\t"
    181     "bne 1f                                \n\t"
    182     "stxr %w[temp], %[new_value], %[ptr]   \n\t"
    183     "cbnz %w[temp], 0b                     \n\t"
    184     "1:                                    \n\t"
    185     : [prev]"=&r" (prev),
    186       [temp]"=&r" (temp),
    187       [ptr]"+Q" (*ptr)
    188     : [old_value]"IJr" (old_value),
    189       [new_value]"r" (new_value)
    190     : "cc", "memory"
    191   );  // NOLINT
    192 
    193   return prev;
    194 }
    195 
    196 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr,
    197                                          Atomic64 new_value) {
    198   Atomic64 result;
    199   int32_t temp;
    200 
    201   __asm__ __volatile__ (  // NOLINT
    202     "0:                                    \n\t"
    203     "ldxr %[result], %[ptr]                \n\t"
    204     "stxr %w[temp], %[new_value], %[ptr]   \n\t"
    205     "cbnz %w[temp], 0b                     \n\t"
    206     : [result]"=&r" (result),
    207       [temp]"=&r" (temp),
    208       [ptr]"+Q" (*ptr)
    209     : [new_value]"r" (new_value)
    210     : "memory"
    211   );  // NOLINT
    212 
    213   return result;
    214 }
    215 
    216 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr,
    217                                           Atomic64 increment) {
    218   Atomic64 result;
    219   int32_t temp;
    220 
    221   __asm__ __volatile__ (  // NOLINT
    222     "0:                                     \n\t"
    223     "ldxr %[result], %[ptr]                 \n\t"
    224     "add %[result], %[result], %[increment] \n\t"
    225     "stxr %w[temp], %[result], %[ptr]       \n\t"
    226     "cbnz %w[temp], 0b                      \n\t"
    227     : [result]"=&r" (result),
    228       [temp]"=&r" (temp),
    229       [ptr]"+Q" (*ptr)
    230     : [increment]"IJr" (increment)
    231     : "memory"
    232   );  // NOLINT
    233 
    234   return result;
    235 }
    236 
    237 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr,
    238                                         Atomic64 increment) {
    239   MemoryBarrier();
    240   Atomic64 result = NoBarrier_AtomicIncrement(ptr, increment);
    241   MemoryBarrier();
    242 
    243   return result;
    244 }
    245 
    246 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr,
    247                                        Atomic64 old_value,
    248                                        Atomic64 new_value) {
    249   Atomic64 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
    250   MemoryBarrier();
    251 
    252   return prev;
    253 }
    254 
    255 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr,
    256                                        Atomic64 old_value,
    257                                        Atomic64 new_value) {
    258   MemoryBarrier();
    259   Atomic64 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
    260 
    261   return prev;
    262 }
    263 
    264 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) {
    265   *ptr = value;
    266 }
    267 
    268 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) {
    269   *ptr = value;
    270   MemoryBarrier();
    271 }
    272 
    273 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) {
    274   __asm__ __volatile__ (  // NOLINT
    275     "stlr %x[value], %[ptr]  \n\t"
    276     : [ptr]"=Q" (*ptr)
    277     : [value]"r" (value)
    278     : "memory"
    279   );  // NOLINT
    280 }
    281 
    282 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) {
    283   return *ptr;
    284 }
    285 
    286 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) {
    287   Atomic64 value;
    288 
    289   __asm__ __volatile__ (  // NOLINT
    290     "ldar %x[value], %[ptr]  \n\t"
    291     : [value]"=r" (value)
    292     : [ptr]"Q" (*ptr)
    293     : "memory"
    294   );  // NOLINT
    295 
    296   return value;
    297 }
    298 
    299 inline Atomic64 Release_Load(volatile const Atomic64* ptr) {
    300   MemoryBarrier();
    301   return *ptr;
    302 }
    303 
    304 }  // namespace base::subtle
    305 }  // namespace base
    306 
    307 #endif  // BASE_ATOMICOPS_INTERNALS_ARM64_GCC_H_
    308