1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_RUNTIME_READ_BARRIER_INL_H_ 18 #define ART_RUNTIME_READ_BARRIER_INL_H_ 19 20 #include "read_barrier.h" 21 22 #include "gc/collector/concurrent_copying.h" 23 #include "gc/heap.h" 24 #include "mirror/object_reference.h" 25 #include "mirror/reference.h" 26 #include "runtime.h" 27 #include "utils.h" 28 29 namespace art { 30 31 template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup> 32 inline MirrorType* ReadBarrier::Barrier( 33 mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr) { 34 const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier; 35 if (with_read_barrier && kUseBakerReadBarrier) { 36 // The higher bits of the rb ptr, rb_ptr_high_bits (must be zero) 37 // is used to create artificial data dependency from the is_gray 38 // load to the ref field (ptr) load to avoid needing a load-load 39 // barrier between the two. 40 uintptr_t rb_ptr_high_bits; 41 bool is_gray = HasGrayReadBarrierPointer(obj, &rb_ptr_high_bits); 42 ref_addr = reinterpret_cast<mirror::HeapReference<MirrorType>*>( 43 rb_ptr_high_bits | reinterpret_cast<uintptr_t>(ref_addr)); 44 MirrorType* ref = ref_addr->AsMirrorPtr(); 45 if (is_gray) { 46 // Slow-path. 47 ref = reinterpret_cast<MirrorType*>(Mark(ref)); 48 } 49 if (kEnableReadBarrierInvariantChecks) { 50 CHECK_EQ(rb_ptr_high_bits, 0U) << obj << " rb_ptr=" << obj->GetReadBarrierPointer(); 51 } 52 AssertToSpaceInvariant(obj, offset, ref); 53 return ref; 54 } else if (with_read_barrier && kUseBrooksReadBarrier) { 55 // To be implemented. 56 return ref_addr->AsMirrorPtr(); 57 } else if (with_read_barrier && kUseTableLookupReadBarrier) { 58 MirrorType* ref = ref_addr->AsMirrorPtr(); 59 MirrorType* old_ref = ref; 60 // The heap or the collector can be null at startup. TODO: avoid the need for this null check. 61 gc::Heap* heap = Runtime::Current()->GetHeap(); 62 if (heap != nullptr && heap->GetReadBarrierTable()->IsSet(old_ref)) { 63 ref = reinterpret_cast<MirrorType*>(Mark(old_ref)); 64 // Update the field atomically. This may fail if mutator updates before us, but it's ok. 65 obj->CasFieldStrongSequentiallyConsistentObjectWithoutWriteBarrier<false, false>( 66 offset, old_ref, ref); 67 } 68 AssertToSpaceInvariant(obj, offset, ref); 69 return ref; 70 } else { 71 // No read barrier. 72 return ref_addr->AsMirrorPtr(); 73 } 74 } 75 76 template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup> 77 inline MirrorType* ReadBarrier::BarrierForRoot(MirrorType** root) { 78 MirrorType* ref = *root; 79 const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier; 80 if (with_read_barrier && kUseBakerReadBarrier) { 81 if (kMaybeDuringStartup && IsDuringStartup()) { 82 // During startup, the heap may not be initialized yet. Just 83 // return the given ref. 84 return ref; 85 } 86 // TODO: separate the read barrier code from the collector code more. 87 if (Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsMarking()) { 88 ref = reinterpret_cast<MirrorType*>(Mark(ref)); 89 } 90 AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 91 return ref; 92 } else if (with_read_barrier && kUseBrooksReadBarrier) { 93 // To be implemented. 94 return ref; 95 } else if (with_read_barrier && kUseTableLookupReadBarrier) { 96 if (kMaybeDuringStartup && IsDuringStartup()) { 97 // During startup, the heap may not be initialized yet. Just 98 // return the given ref. 99 return ref; 100 } 101 if (Runtime::Current()->GetHeap()->GetReadBarrierTable()->IsSet(ref)) { 102 MirrorType* old_ref = ref; 103 ref = reinterpret_cast<MirrorType*>(Mark(old_ref)); 104 // Update the field atomically. This may fail if mutator updates before us, but it's ok. 105 Atomic<mirror::Object*>* atomic_root = reinterpret_cast<Atomic<mirror::Object*>*>(root); 106 atomic_root->CompareExchangeStrongSequentiallyConsistent(old_ref, ref); 107 } 108 AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 109 return ref; 110 } else { 111 return ref; 112 } 113 } 114 115 // TODO: Reduce copy paste 116 template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup> 117 inline MirrorType* ReadBarrier::BarrierForRoot(mirror::CompressedReference<MirrorType>* root) { 118 MirrorType* ref = root->AsMirrorPtr(); 119 const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier; 120 if (with_read_barrier && kUseBakerReadBarrier) { 121 if (kMaybeDuringStartup && IsDuringStartup()) { 122 // During startup, the heap may not be initialized yet. Just 123 // return the given ref. 124 return ref; 125 } 126 // TODO: separate the read barrier code from the collector code more. 127 if (Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsMarking()) { 128 ref = reinterpret_cast<MirrorType*>(Mark(ref)); 129 } 130 AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 131 return ref; 132 } else if (with_read_barrier && kUseBrooksReadBarrier) { 133 // To be implemented. 134 return ref; 135 } else if (with_read_barrier && kUseTableLookupReadBarrier) { 136 if (kMaybeDuringStartup && IsDuringStartup()) { 137 // During startup, the heap may not be initialized yet. Just 138 // return the given ref. 139 return ref; 140 } 141 if (Runtime::Current()->GetHeap()->GetReadBarrierTable()->IsSet(ref)) { 142 auto old_ref = mirror::CompressedReference<MirrorType>::FromMirrorPtr(ref); 143 ref = reinterpret_cast<MirrorType*>(Mark(ref)); 144 auto new_ref = mirror::CompressedReference<MirrorType>::FromMirrorPtr(ref); 145 // Update the field atomically. This may fail if mutator updates before us, but it's ok. 146 auto* atomic_root = 147 reinterpret_cast<Atomic<mirror::CompressedReference<MirrorType>>*>(root); 148 atomic_root->CompareExchangeStrongSequentiallyConsistent(old_ref, new_ref); 149 } 150 AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 151 return ref; 152 } else { 153 return ref; 154 } 155 } 156 157 inline bool ReadBarrier::IsDuringStartup() { 158 gc::Heap* heap = Runtime::Current()->GetHeap(); 159 if (heap == nullptr) { 160 // During startup, the heap can be null. 161 return true; 162 } 163 if (heap->CurrentCollectorType() != gc::kCollectorTypeCC) { 164 // CC isn't running. 165 return true; 166 } 167 gc::collector::ConcurrentCopying* collector = heap->ConcurrentCopyingCollector(); 168 if (collector == nullptr) { 169 // During startup, the collector can be null. 170 return true; 171 } 172 return false; 173 } 174 175 inline void ReadBarrier::AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset, 176 mirror::Object* ref) { 177 if (kEnableToSpaceInvariantChecks || kIsDebugBuild) { 178 if (ref == nullptr || IsDuringStartup()) { 179 return; 180 } 181 Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()-> 182 AssertToSpaceInvariant(obj, offset, ref); 183 } 184 } 185 186 inline mirror::Object* ReadBarrier::Mark(mirror::Object* obj) { 187 return Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->Mark(obj); 188 } 189 190 inline bool ReadBarrier::HasGrayReadBarrierPointer(mirror::Object* obj, 191 uintptr_t* out_rb_ptr_high_bits) { 192 mirror::Object* rb_ptr = obj->GetReadBarrierPointer(); 193 uintptr_t rb_ptr_bits = reinterpret_cast<uintptr_t>(rb_ptr); 194 uintptr_t rb_ptr_low_bits = rb_ptr_bits & rb_ptr_mask_; 195 if (kEnableReadBarrierInvariantChecks) { 196 CHECK(rb_ptr_low_bits == white_ptr_ || rb_ptr_low_bits == gray_ptr_ || 197 rb_ptr_low_bits == black_ptr_) 198 << "obj=" << obj << " rb_ptr=" << rb_ptr << " " << PrettyTypeOf(obj); 199 } 200 bool is_gray = rb_ptr_low_bits == gray_ptr_; 201 // The high bits are supposed to be zero. We check this on the caller side. 202 *out_rb_ptr_high_bits = rb_ptr_bits & ~rb_ptr_mask_; 203 return is_gray; 204 } 205 206 } // namespace art 207 208 #endif // ART_RUNTIME_READ_BARRIER_INL_H_ 209
