1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_ 18 #define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_ 19 20 #include "object.h" 21 22 #include "atomic.h" 23 #include "lock_word-inl.h" 24 #include "object_reference-inl.h" 25 #include "read_barrier.h" 26 #include "runtime.h" 27 28 namespace art { 29 namespace mirror { 30 31 template<VerifyObjectFlags kVerifyFlags> 32 inline LockWord Object::GetLockWord(bool as_volatile) { 33 if (as_volatile) { 34 return LockWord(GetField32Volatile<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 35 } 36 return LockWord(GetField32<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 37 } 38 39 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> 40 inline bool Object::CasFieldWeakRelaxed32(MemberOffset field_offset, 41 int32_t old_value, int32_t new_value) { 42 if (kCheckTransaction) { 43 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); 44 } 45 if (kTransactionActive) { 46 Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true); 47 } 48 if (kVerifyFlags & kVerifyThis) { 49 VerifyObject(this); 50 } 51 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); 52 AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr); 53 54 return atomic_addr->CompareExchangeWeakRelaxed(old_value, new_value); 55 } 56 57 inline bool Object::CasLockWordWeakRelaxed(LockWord old_val, LockWord new_val) { 58 // Force use of non-transactional mode and do not check. 59 return CasFieldWeakRelaxed32<false, false>( 60 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue()); 61 } 62 63 inline bool Object::CasLockWordWeakRelease(LockWord old_val, LockWord new_val) { 64 // Force use of non-transactional mode and do not check. 65 return CasFieldWeakRelease32<false, false>( 66 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue()); 67 } 68 69 inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) { 70 if (!kUseBakerReadBarrier) { 71 LOG(FATAL) << "Unreachable"; 72 UNREACHABLE(); 73 } 74 #if defined(__arm__) 75 uintptr_t obj = reinterpret_cast<uintptr_t>(this); 76 uintptr_t result; 77 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U); 78 // Use inline assembly to prevent the compiler from optimizing away the false dependency. 79 __asm__ __volatile__( 80 "ldr %[result], [%[obj], #4]\n\t" 81 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be 82 // null, without them being able to assume that fact. 83 "eor %[fad], %[result], %[result]\n\t" 84 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency) 85 : [obj] "r" (obj)); 86 DCHECK_EQ(*fake_address_dependency, 0U); 87 LockWord lw(static_cast<uint32_t>(result)); 88 uint32_t rb_state = lw.ReadBarrierState(); 89 return rb_state; 90 #elif defined(__aarch64__) 91 uintptr_t obj = reinterpret_cast<uintptr_t>(this); 92 uintptr_t result; 93 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U); 94 // Use inline assembly to prevent the compiler from optimizing away the false dependency. 95 __asm__ __volatile__( 96 "ldr %w[result], [%[obj], #4]\n\t" 97 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be 98 // null, without them being able to assume that fact. 99 "eor %[fad], %[result], %[result]\n\t" 100 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency) 101 : [obj] "r" (obj)); 102 DCHECK_EQ(*fake_address_dependency, 0U); 103 LockWord lw(static_cast<uint32_t>(result)); 104 uint32_t rb_state = lw.ReadBarrierState(); 105 return rb_state; 106 #elif defined(__i386__) || defined(__x86_64__) 107 LockWord lw = GetLockWord(false); 108 // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler 109 // reordering. 110 *fake_address_dependency = 0; 111 std::atomic_signal_fence(std::memory_order_acquire); 112 uint32_t rb_state = lw.ReadBarrierState(); 113 return rb_state; 114 #else 115 // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering. 116 LockWord lw = GetLockWord(false); 117 *fake_address_dependency = 0; 118 std::atomic_thread_fence(std::memory_order_acquire); 119 uint32_t rb_state = lw.ReadBarrierState(); 120 return rb_state; 121 #endif 122 } 123 124 inline uint32_t Object::GetReadBarrierState() { 125 if (!kUseBakerReadBarrier) { 126 LOG(FATAL) << "Unreachable"; 127 UNREACHABLE(); 128 } 129 DCHECK(kUseBakerReadBarrier); 130 LockWord lw(GetField<uint32_t, /*kIsVolatile*/false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 131 uint32_t rb_state = lw.ReadBarrierState(); 132 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; 133 return rb_state; 134 } 135 136 inline uint32_t Object::GetReadBarrierStateAcquire() { 137 if (!kUseBakerReadBarrier) { 138 LOG(FATAL) << "Unreachable"; 139 UNREACHABLE(); 140 } 141 LockWord lw(GetFieldAcquire<uint32_t>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 142 uint32_t rb_state = lw.ReadBarrierState(); 143 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; 144 return rb_state; 145 } 146 147 template<bool kCasRelease> 148 inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) { 149 if (!kUseBakerReadBarrier) { 150 LOG(FATAL) << "Unreachable"; 151 UNREACHABLE(); 152 } 153 DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state; 154 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; 155 LockWord expected_lw; 156 LockWord new_lw; 157 do { 158 LockWord lw = GetLockWord(false); 159 if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) { 160 // Lost the race. 161 return false; 162 } 163 expected_lw = lw; 164 expected_lw.SetReadBarrierState(expected_rb_state); 165 new_lw = lw; 166 new_lw.SetReadBarrierState(rb_state); 167 // ConcurrentCopying::ProcessMarkStackRef uses this with kCasRelease == true. 168 // If kCasRelease == true, use a CAS release so that when GC updates all the fields of 169 // an object and then changes the object from gray to black, the field updates (stores) will be 170 // visible (won't be reordered after this CAS.) 171 } while (!(kCasRelease ? 172 CasLockWordWeakRelease(expected_lw, new_lw) : 173 CasLockWordWeakRelaxed(expected_lw, new_lw))); 174 return true; 175 } 176 177 inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) { 178 LockWord expected_lw; 179 LockWord new_lw; 180 do { 181 LockWord lw = GetLockWord(false); 182 if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) { 183 // Lost the race. 184 return false; 185 } 186 expected_lw = lw; 187 new_lw = lw; 188 new_lw.SetMarkBitState(mark_bit); 189 // Since this is only set from the mutator, we can use the non release Cas. 190 } while (!CasLockWordWeakRelaxed(expected_lw, new_lw)); 191 return true; 192 } 193 194 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> 195 inline bool Object::CasFieldStrongRelaxedObjectWithoutWriteBarrier( 196 MemberOffset field_offset, 197 ObjPtr<Object> old_value, 198 ObjPtr<Object> new_value) { 199 if (kCheckTransaction) { 200 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); 201 } 202 if (kVerifyFlags & kVerifyThis) { 203 VerifyObject(this); 204 } 205 if (kVerifyFlags & kVerifyWrites) { 206 VerifyObject(new_value); 207 } 208 if (kVerifyFlags & kVerifyReads) { 209 VerifyObject(old_value); 210 } 211 if (kTransactionActive) { 212 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); 213 } 214 HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); 215 HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); 216 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); 217 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); 218 219 bool success = atomic_addr->CompareExchangeStrongRelaxed(old_ref.reference_, 220 new_ref.reference_); 221 return success; 222 } 223 224 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> 225 inline bool Object::CasFieldStrongReleaseObjectWithoutWriteBarrier( 226 MemberOffset field_offset, 227 ObjPtr<Object> old_value, 228 ObjPtr<Object> new_value) { 229 if (kCheckTransaction) { 230 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); 231 } 232 if (kVerifyFlags & kVerifyThis) { 233 VerifyObject(this); 234 } 235 if (kVerifyFlags & kVerifyWrites) { 236 VerifyObject(new_value); 237 } 238 if (kVerifyFlags & kVerifyReads) { 239 VerifyObject(old_value); 240 } 241 if (kTransactionActive) { 242 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); 243 } 244 HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); 245 HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); 246 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); 247 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); 248 249 bool success = atomic_addr->CompareExchangeStrongRelease(old_ref.reference_, 250 new_ref.reference_); 251 return success; 252 } 253 254 } // namespace mirror 255 } // namespace art 256 257 #endif // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_ 258
