1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "atomic.h" 18 19 #define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__) 20 21 #if NEED_SWAP_MUTEXES 22 #include <vector> 23 #include "base/mutex.h" 24 #include "base/stl_util.h" 25 #include "base/stringprintf.h" 26 #include "thread.h" 27 #endif 28 29 namespace art { 30 31 #if NEED_SWAP_MUTEXES 32 // We stripe across a bunch of different mutexes to reduce contention. 33 static const size_t kSwapMutexCount = 32; 34 static std::vector<Mutex*>* gSwapMutexes; 35 36 static Mutex& GetSwapMutex(const volatile int64_t* addr) { 37 return *(*gSwapMutexes)[(reinterpret_cast<unsigned>(addr) >> 3U) % kSwapMutexCount]; 38 } 39 #endif 40 41 void QuasiAtomic::Startup() { 42 #if NEED_SWAP_MUTEXES 43 gSwapMutexes = new std::vector<Mutex*>; 44 for (size_t i = 0; i < kSwapMutexCount; ++i) { 45 gSwapMutexes->push_back(new Mutex("QuasiAtomic stripe")); 46 } 47 #endif 48 } 49 50 void QuasiAtomic::Shutdown() { 51 #if NEED_SWAP_MUTEXES 52 STLDeleteElements(gSwapMutexes); 53 delete gSwapMutexes; 54 #endif 55 } 56 57 int64_t QuasiAtomic::Read64(volatile const int64_t* addr) { 58 int64_t value; 59 #if NEED_SWAP_MUTEXES 60 MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 61 value = *addr; 62 #elif defined(__arm__) 63 // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we 64 // have LPAE (such as Cortex-A15) then ldrd would suffice. 65 __asm__ __volatile__("@ QuasiAtomic::Read64\n" 66 "ldrexd %0, %H0, [%1]" 67 : "=&r" (value) 68 : "r" (addr)); 69 #elif defined(__i386__) 70 __asm__ __volatile__( 71 "movq %1, %0\n" 72 : "=x" (value) 73 : "m" (*addr)); 74 #else 75 #error Unexpected architecture 76 #endif 77 return value; 78 } 79 80 void QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) { 81 #if NEED_SWAP_MUTEXES 82 MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 83 *addr = value; 84 #elif defined(__arm__) 85 // The write is done as a swap so that the cache-line is in the exclusive state for the store. If 86 // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will 87 // suffice. 88 int64_t prev; 89 int status; 90 do { 91 __asm__ __volatile__("@ QuasiAtomic::Write64\n" 92 "ldrexd %0, %H0, [%3]\n" 93 "strexd %1, %4, %H4, [%3]" 94 : "=&r" (prev), "=&r" (status), "+m"(*addr) 95 : "r" (addr), "r" (value) 96 : "cc"); 97 } while (__builtin_expect(status != 0, 0)); 98 #elif defined(__i386__) 99 __asm__ __volatile__( 100 "movq %1, %0" 101 : "=m" (*addr) 102 : "x" (value)); 103 #else 104 #error Unexpected architecture 105 #endif 106 } 107 108 109 bool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 110 #if NEED_SWAP_MUTEXES 111 MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 112 if (*addr == old_value) { 113 *addr = new_value; 114 return true; 115 } 116 return false; 117 #elif defined(__arm__) 118 int64_t prev; 119 int status; 120 do { 121 __asm__ __volatile__("@ QuasiAtomic::Cas64\n" 122 "ldrexd %0, %H0, [%3]\n" 123 "mov %1, #0\n" 124 "teq %0, %4\n" 125 "teqeq %H0, %H4\n" 126 "strexdeq %1, %5, %H5, [%3]" 127 : "=&r" (prev), "=&r" (status), "+m"(*addr) 128 : "r" (addr), "Ir" (old_value), "r" (new_value) 129 : "cc"); 130 } while (__builtin_expect(status != 0, 0)); 131 return prev == old_value; 132 #elif defined(__i386__) 133 // The compiler does the right job and works better than inline assembly, especially with -O0 134 // compilation. 135 return __sync_bool_compare_and_swap(addr, old_value, new_value); 136 #else 137 #error Unexpected architecture 138 #endif 139 } 140 141 bool QuasiAtomic::LongAtomicsUseMutexes() { 142 #if NEED_SWAP_MUTEXES 143 return true; 144 #else 145 return false; 146 #endif 147 } 148 149 } // namespace art 150
