1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_ 18 #define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_ 19 20 #include "object.h" 21 22 #include "base/atomic.h" 23 #include "heap_poisoning.h" 24 #include "lock_word-inl.h" 25 #include "object_reference-inl.h" 26 #include "read_barrier.h" 27 #include "runtime.h" 28 29 namespace art { 30 namespace mirror { 31 32 template<VerifyObjectFlags kVerifyFlags> 33 inline LockWord Object::GetLockWord(bool as_volatile) { 34 if (as_volatile) { 35 return LockWord(GetField32Volatile<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 36 } 37 return LockWord(GetField32<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 38 } 39 40 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> 41 inline bool Object::CasFieldWeakRelaxed32(MemberOffset field_offset, 42 int32_t old_value, int32_t new_value) { 43 if (kCheckTransaction) { 44 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); 45 } 46 if (kTransactionActive) { 47 Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true); 48 } 49 if (kVerifyFlags & kVerifyThis) { 50 VerifyObject(this); 51 } 52 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); 53 AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr); 54 55 return atomic_addr->CompareAndSetWeakRelaxed(old_value, new_value); 56 } 57 58 inline bool Object::CasLockWordWeakRelaxed(LockWord old_val, LockWord new_val) { 59 // Force use of non-transactional mode and do not check. 60 return CasFieldWeakRelaxed32<false, false>( 61 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue()); 62 } 63 64 inline bool Object::CasLockWordWeakRelease(LockWord old_val, LockWord new_val) { 65 // Force use of non-transactional mode and do not check. 66 return CasFieldWeakRelease32<false, false>( 67 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue()); 68 } 69 70 inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) { 71 if (!kUseBakerReadBarrier) { 72 LOG(FATAL) << "Unreachable"; 73 UNREACHABLE(); 74 } 75 #if defined(__arm__) 76 uintptr_t obj = reinterpret_cast<uintptr_t>(this); 77 uintptr_t result; 78 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U); 79 // Use inline assembly to prevent the compiler from optimizing away the false dependency. 80 __asm__ __volatile__( 81 "ldr %[result], [%[obj], #4]\n\t" 82 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be 83 // null, without them being able to assume that fact. 84 "eor %[fad], %[result], %[result]\n\t" 85 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency) 86 : [obj] "r" (obj)); 87 DCHECK_EQ(*fake_address_dependency, 0U); 88 LockWord lw(static_cast<uint32_t>(result)); 89 uint32_t rb_state = lw.ReadBarrierState(); 90 return rb_state; 91 #elif defined(__aarch64__) 92 uintptr_t obj = reinterpret_cast<uintptr_t>(this); 93 uintptr_t result; 94 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U); 95 // Use inline assembly to prevent the compiler from optimizing away the false dependency. 96 __asm__ __volatile__( 97 "ldr %w[result], [%[obj], #4]\n\t" 98 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be 99 // null, without them being able to assume that fact. 100 "eor %[fad], %[result], %[result]\n\t" 101 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency) 102 : [obj] "r" (obj)); 103 DCHECK_EQ(*fake_address_dependency, 0U); 104 LockWord lw(static_cast<uint32_t>(result)); 105 uint32_t rb_state = lw.ReadBarrierState(); 106 return rb_state; 107 #elif defined(__i386__) || defined(__x86_64__) 108 LockWord lw = GetLockWord(false); 109 // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler 110 // reordering. 111 *fake_address_dependency = 0; 112 std::atomic_signal_fence(std::memory_order_acquire); 113 uint32_t rb_state = lw.ReadBarrierState(); 114 return rb_state; 115 #else 116 // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering. 117 LockWord lw = GetLockWord(false); 118 *fake_address_dependency = 0; 119 std::atomic_thread_fence(std::memory_order_acquire); 120 uint32_t rb_state = lw.ReadBarrierState(); 121 return rb_state; 122 #endif 123 } 124 125 inline uint32_t Object::GetReadBarrierState() { 126 if (!kUseBakerReadBarrier) { 127 LOG(FATAL) << "Unreachable"; 128 UNREACHABLE(); 129 } 130 DCHECK(kUseBakerReadBarrier); 131 LockWord lw(GetField<uint32_t, /*kIsVolatile*/false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 132 uint32_t rb_state = lw.ReadBarrierState(); 133 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; 134 return rb_state; 135 } 136 137 inline uint32_t Object::GetReadBarrierStateAcquire() { 138 if (!kUseBakerReadBarrier) { 139 LOG(FATAL) << "Unreachable"; 140 UNREACHABLE(); 141 } 142 LockWord lw(GetFieldAcquire<uint32_t>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); 143 uint32_t rb_state = lw.ReadBarrierState(); 144 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; 145 return rb_state; 146 } 147 148 template<bool kCasRelease> 149 inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) { 150 if (!kUseBakerReadBarrier) { 151 LOG(FATAL) << "Unreachable"; 152 UNREACHABLE(); 153 } 154 DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state; 155 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; 156 LockWord expected_lw; 157 LockWord new_lw; 158 do { 159 LockWord lw = GetLockWord(false); 160 if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) { 161 // Lost the race. 162 return false; 163 } 164 expected_lw = lw; 165 expected_lw.SetReadBarrierState(expected_rb_state); 166 new_lw = lw; 167 new_lw.SetReadBarrierState(rb_state); 168 // ConcurrentCopying::ProcessMarkStackRef uses this with kCasRelease == true. 169 // If kCasRelease == true, use a CAS release so that when GC updates all the fields of 170 // an object and then changes the object from gray to black, the field updates (stores) will be 171 // visible (won't be reordered after this CAS.) 172 } while (!(kCasRelease ? 173 CasLockWordWeakRelease(expected_lw, new_lw) : 174 CasLockWordWeakRelaxed(expected_lw, new_lw))); 175 return true; 176 } 177 178 inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) { 179 LockWord expected_lw; 180 LockWord new_lw; 181 do { 182 LockWord lw = GetLockWord(false); 183 if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) { 184 // Lost the race. 185 return false; 186 } 187 expected_lw = lw; 188 new_lw = lw; 189 new_lw.SetMarkBitState(mark_bit); 190 // Since this is only set from the mutator, we can use the non-release CAS. 191 } while (!CasLockWordWeakRelaxed(expected_lw, new_lw)); 192 return true; 193 } 194 195 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> 196 inline bool Object::CasFieldStrongRelaxedObjectWithoutWriteBarrier( 197 MemberOffset field_offset, 198 ObjPtr<Object> old_value, 199 ObjPtr<Object> new_value) { 200 if (kCheckTransaction) { 201 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); 202 } 203 if (kVerifyFlags & kVerifyThis) { 204 VerifyObject(this); 205 } 206 if (kVerifyFlags & kVerifyWrites) { 207 VerifyObject(new_value); 208 } 209 if (kVerifyFlags & kVerifyReads) { 210 VerifyObject(old_value); 211 } 212 if (kTransactionActive) { 213 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); 214 } 215 uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); 216 uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); 217 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); 218 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); 219 220 bool success = atomic_addr->CompareAndSetStrongRelaxed(old_ref, new_ref); 221 return success; 222 } 223 224 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> 225 inline bool Object::CasFieldStrongReleaseObjectWithoutWriteBarrier( 226 MemberOffset field_offset, 227 ObjPtr<Object> old_value, 228 ObjPtr<Object> new_value) { 229 if (kCheckTransaction) { 230 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); 231 } 232 if (kVerifyFlags & kVerifyThis) { 233 VerifyObject(this); 234 } 235 if (kVerifyFlags & kVerifyWrites) { 236 VerifyObject(new_value); 237 } 238 if (kVerifyFlags & kVerifyReads) { 239 VerifyObject(old_value); 240 } 241 if (kTransactionActive) { 242 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); 243 } 244 uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); 245 uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); 246 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); 247 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); 248 249 bool success = atomic_addr->CompareAndSetStrongRelease(old_ref, new_ref); 250 return success; 251 } 252 253 } // namespace mirror 254 } // namespace art 255 256 #endif // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_ 257
