1 //=-- InstrProfReader.cpp - Instrumented profiling reader -------------------=// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains support for reading profiling data for clang's 11 // instrumentation based PGO and coverage. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/ProfileData/InstrProfReader.h" 16 #include "InstrProfIndexed.h" 17 #include "llvm/ADT/STLExtras.h" 18 #include "llvm/ProfileData/InstrProf.h" 19 #include <cassert> 20 21 using namespace llvm; 22 23 static ErrorOr<std::unique_ptr<MemoryBuffer>> 24 setupMemoryBuffer(std::string Path) { 25 ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = 26 MemoryBuffer::getFileOrSTDIN(Path); 27 if (std::error_code EC = BufferOrErr.getError()) 28 return EC; 29 return std::move(BufferOrErr.get()); 30 } 31 32 static std::error_code initializeReader(InstrProfReader &Reader) { 33 return Reader.readHeader(); 34 } 35 36 ErrorOr<std::unique_ptr<InstrProfReader>> 37 InstrProfReader::create(std::string Path) { 38 // Set up the buffer to read. 39 auto BufferOrError = setupMemoryBuffer(Path); 40 if (std::error_code EC = BufferOrError.getError()) 41 return EC; 42 return InstrProfReader::create(std::move(BufferOrError.get())); 43 } 44 45 ErrorOr<std::unique_ptr<InstrProfReader>> 46 InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { 47 // Sanity check the buffer. 48 if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max()) 49 return instrprof_error::too_large; 50 51 std::unique_ptr<InstrProfReader> Result; 52 // Create the reader. 53 if (IndexedInstrProfReader::hasFormat(*Buffer)) 54 Result.reset(new IndexedInstrProfReader(std::move(Buffer))); 55 else if (RawInstrProfReader64::hasFormat(*Buffer)) 56 Result.reset(new RawInstrProfReader64(std::move(Buffer))); 57 else if (RawInstrProfReader32::hasFormat(*Buffer)) 58 Result.reset(new RawInstrProfReader32(std::move(Buffer))); 59 else 60 Result.reset(new TextInstrProfReader(std::move(Buffer))); 61 62 // Initialize the reader and return the result. 63 if (std::error_code EC = initializeReader(*Result)) 64 return EC; 65 66 return std::move(Result); 67 } 68 69 ErrorOr<std::unique_ptr<IndexedInstrProfReader>> 70 IndexedInstrProfReader::create(std::string Path) { 71 // Set up the buffer to read. 72 auto BufferOrError = setupMemoryBuffer(Path); 73 if (std::error_code EC = BufferOrError.getError()) 74 return EC; 75 return IndexedInstrProfReader::create(std::move(BufferOrError.get())); 76 } 77 78 79 ErrorOr<std::unique_ptr<IndexedInstrProfReader>> 80 IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { 81 // Sanity check the buffer. 82 if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max()) 83 return instrprof_error::too_large; 84 85 // Create the reader. 86 if (!IndexedInstrProfReader::hasFormat(*Buffer)) 87 return instrprof_error::bad_magic; 88 auto Result = llvm::make_unique<IndexedInstrProfReader>(std::move(Buffer)); 89 90 // Initialize the reader and return the result. 91 if (std::error_code EC = initializeReader(*Result)) 92 return EC; 93 94 return std::move(Result); 95 } 96 97 void InstrProfIterator::Increment() { 98 if (Reader->readNextRecord(Record)) 99 *this = InstrProfIterator(); 100 } 101 102 std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { 103 // Skip empty lines and comments. 104 while (!Line.is_at_end() && (Line->empty() || Line->startswith("#"))) 105 ++Line; 106 // If we hit EOF while looking for a name, we're done. 107 if (Line.is_at_end()) 108 return error(instrprof_error::eof); 109 110 // Read the function name. 111 Record.Name = *Line++; 112 113 // Read the function hash. 114 if (Line.is_at_end()) 115 return error(instrprof_error::truncated); 116 if ((Line++)->getAsInteger(0, Record.Hash)) 117 return error(instrprof_error::malformed); 118 119 // Read the number of counters. 120 uint64_t NumCounters; 121 if (Line.is_at_end()) 122 return error(instrprof_error::truncated); 123 if ((Line++)->getAsInteger(10, NumCounters)) 124 return error(instrprof_error::malformed); 125 if (NumCounters == 0) 126 return error(instrprof_error::malformed); 127 128 // Read each counter and fill our internal storage with the values. 129 Counts.clear(); 130 Counts.reserve(NumCounters); 131 for (uint64_t I = 0; I < NumCounters; ++I) { 132 if (Line.is_at_end()) 133 return error(instrprof_error::truncated); 134 uint64_t Count; 135 if ((Line++)->getAsInteger(10, Count)) 136 return error(instrprof_error::malformed); 137 Counts.push_back(Count); 138 } 139 // Give the record a reference to our internal counter storage. 140 Record.Counts = Counts; 141 142 return success(); 143 } 144 145 template <class IntPtrT> 146 static uint64_t getRawMagic(); 147 148 template <> 149 uint64_t getRawMagic<uint64_t>() { 150 return 151 uint64_t(255) << 56 | 152 uint64_t('l') << 48 | 153 uint64_t('p') << 40 | 154 uint64_t('r') << 32 | 155 uint64_t('o') << 24 | 156 uint64_t('f') << 16 | 157 uint64_t('r') << 8 | 158 uint64_t(129); 159 } 160 161 template <> 162 uint64_t getRawMagic<uint32_t>() { 163 return 164 uint64_t(255) << 56 | 165 uint64_t('l') << 48 | 166 uint64_t('p') << 40 | 167 uint64_t('r') << 32 | 168 uint64_t('o') << 24 | 169 uint64_t('f') << 16 | 170 uint64_t('R') << 8 | 171 uint64_t(129); 172 } 173 174 template <class IntPtrT> 175 bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) { 176 if (DataBuffer.getBufferSize() < sizeof(uint64_t)) 177 return false; 178 uint64_t Magic = 179 *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart()); 180 return getRawMagic<IntPtrT>() == Magic || 181 sys::getSwappedBytes(getRawMagic<IntPtrT>()) == Magic; 182 } 183 184 template <class IntPtrT> 185 std::error_code RawInstrProfReader<IntPtrT>::readHeader() { 186 if (!hasFormat(*DataBuffer)) 187 return error(instrprof_error::bad_magic); 188 if (DataBuffer->getBufferSize() < sizeof(RawHeader)) 189 return error(instrprof_error::bad_header); 190 auto *Header = 191 reinterpret_cast<const RawHeader *>(DataBuffer->getBufferStart()); 192 ShouldSwapBytes = Header->Magic != getRawMagic<IntPtrT>(); 193 return readHeader(*Header); 194 } 195 196 template <class IntPtrT> 197 std::error_code 198 RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) { 199 const char *End = DataBuffer->getBufferEnd(); 200 // Skip zero padding between profiles. 201 while (CurrentPos != End && *CurrentPos == 0) 202 ++CurrentPos; 203 // If there's nothing left, we're done. 204 if (CurrentPos == End) 205 return instrprof_error::eof; 206 // If there isn't enough space for another header, this is probably just 207 // garbage at the end of the file. 208 if (CurrentPos + sizeof(RawHeader) > End) 209 return instrprof_error::malformed; 210 // The writer ensures each profile is padded to start at an aligned address. 211 if (reinterpret_cast<size_t>(CurrentPos) % alignOf<uint64_t>()) 212 return instrprof_error::malformed; 213 // The magic should have the same byte order as in the previous header. 214 uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos); 215 if (Magic != swap(getRawMagic<IntPtrT>())) 216 return instrprof_error::bad_magic; 217 218 // There's another profile to read, so we need to process the header. 219 auto *Header = reinterpret_cast<const RawHeader *>(CurrentPos); 220 return readHeader(*Header); 221 } 222 223 static uint64_t getRawVersion() { 224 return 1; 225 } 226 227 template <class IntPtrT> 228 std::error_code 229 RawInstrProfReader<IntPtrT>::readHeader(const RawHeader &Header) { 230 if (swap(Header.Version) != getRawVersion()) 231 return error(instrprof_error::unsupported_version); 232 233 CountersDelta = swap(Header.CountersDelta); 234 NamesDelta = swap(Header.NamesDelta); 235 auto DataSize = swap(Header.DataSize); 236 auto CountersSize = swap(Header.CountersSize); 237 auto NamesSize = swap(Header.NamesSize); 238 239 ptrdiff_t DataOffset = sizeof(RawHeader); 240 ptrdiff_t CountersOffset = DataOffset + sizeof(ProfileData) * DataSize; 241 ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize; 242 size_t ProfileSize = NamesOffset + sizeof(char) * NamesSize; 243 244 auto *Start = reinterpret_cast<const char *>(&Header); 245 if (Start + ProfileSize > DataBuffer->getBufferEnd()) 246 return error(instrprof_error::bad_header); 247 248 Data = reinterpret_cast<const ProfileData *>(Start + DataOffset); 249 DataEnd = Data + DataSize; 250 CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset); 251 NamesStart = Start + NamesOffset; 252 ProfileEnd = Start + ProfileSize; 253 254 return success(); 255 } 256 257 template <class IntPtrT> 258 std::error_code 259 RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) { 260 if (Data == DataEnd) 261 if (std::error_code EC = readNextHeader(ProfileEnd)) 262 return EC; 263 264 // Get the raw data. 265 StringRef RawName(getName(Data->NamePtr), swap(Data->NameSize)); 266 uint32_t NumCounters = swap(Data->NumCounters); 267 if (NumCounters == 0) 268 return error(instrprof_error::malformed); 269 auto RawCounts = makeArrayRef(getCounter(Data->CounterPtr), NumCounters); 270 271 // Check bounds. 272 auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart); 273 if (RawName.data() < NamesStart || 274 RawName.data() + RawName.size() > DataBuffer->getBufferEnd() || 275 RawCounts.data() < CountersStart || 276 RawCounts.data() + RawCounts.size() > NamesStartAsCounter) 277 return error(instrprof_error::malformed); 278 279 // Store the data in Record, byte-swapping as necessary. 280 Record.Hash = swap(Data->FuncHash); 281 Record.Name = RawName; 282 if (ShouldSwapBytes) { 283 Counts.clear(); 284 Counts.reserve(RawCounts.size()); 285 for (uint64_t Count : RawCounts) 286 Counts.push_back(swap(Count)); 287 Record.Counts = Counts; 288 } else 289 Record.Counts = RawCounts; 290 291 // Iterate. 292 ++Data; 293 return success(); 294 } 295 296 namespace llvm { 297 template class RawInstrProfReader<uint32_t>; 298 template class RawInstrProfReader<uint64_t>; 299 } 300 301 InstrProfLookupTrait::hash_value_type 302 InstrProfLookupTrait::ComputeHash(StringRef K) { 303 return IndexedInstrProf::ComputeHash(HashType, K); 304 } 305 306 bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { 307 if (DataBuffer.getBufferSize() < 8) 308 return false; 309 using namespace support; 310 uint64_t Magic = 311 endian::read<uint64_t, little, aligned>(DataBuffer.getBufferStart()); 312 return Magic == IndexedInstrProf::Magic; 313 } 314 315 std::error_code IndexedInstrProfReader::readHeader() { 316 const unsigned char *Start = 317 (const unsigned char *)DataBuffer->getBufferStart(); 318 const unsigned char *Cur = Start; 319 if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24) 320 return error(instrprof_error::truncated); 321 322 using namespace support; 323 324 // Check the magic number. 325 uint64_t Magic = endian::readNext<uint64_t, little, unaligned>(Cur); 326 if (Magic != IndexedInstrProf::Magic) 327 return error(instrprof_error::bad_magic); 328 329 // Read the version. 330 FormatVersion = endian::readNext<uint64_t, little, unaligned>(Cur); 331 if (FormatVersion > IndexedInstrProf::Version) 332 return error(instrprof_error::unsupported_version); 333 334 // Read the maximal function count. 335 MaxFunctionCount = endian::readNext<uint64_t, little, unaligned>(Cur); 336 337 // Read the hash type and start offset. 338 IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>( 339 endian::readNext<uint64_t, little, unaligned>(Cur)); 340 if (HashType > IndexedInstrProf::HashT::Last) 341 return error(instrprof_error::unsupported_hash_type); 342 uint64_t HashOffset = endian::readNext<uint64_t, little, unaligned>(Cur); 343 344 // The rest of the file is an on disk hash table. 345 Index.reset(InstrProfReaderIndex::Create(Start + HashOffset, Cur, Start, 346 InstrProfLookupTrait(HashType))); 347 // Set up our iterator for readNextRecord. 348 RecordIterator = Index->data_begin(); 349 350 return success(); 351 } 352 353 std::error_code IndexedInstrProfReader::getFunctionCounts( 354 StringRef FuncName, uint64_t FuncHash, std::vector<uint64_t> &Counts) { 355 auto Iter = Index->find(FuncName); 356 if (Iter == Index->end()) 357 return error(instrprof_error::unknown_function); 358 359 // Found it. Look for counters with the right hash. 360 ArrayRef<uint64_t> Data = (*Iter).Data; 361 uint64_t NumCounts; 362 for (uint64_t I = 0, E = Data.size(); I != E; I += NumCounts) { 363 // The function hash comes first. 364 uint64_t FoundHash = Data[I++]; 365 // In v1, we have at least one count. Later, we have the number of counts. 366 if (I == E) 367 return error(instrprof_error::malformed); 368 NumCounts = FormatVersion == 1 ? E - I : Data[I++]; 369 // If we have more counts than data, this is bogus. 370 if (I + NumCounts > E) 371 return error(instrprof_error::malformed); 372 // Check for a match and fill the vector if there is one. 373 if (FoundHash == FuncHash) { 374 Counts = Data.slice(I, NumCounts); 375 return success(); 376 } 377 } 378 return error(instrprof_error::hash_mismatch); 379 } 380 381 std::error_code 382 IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) { 383 // Are we out of records? 384 if (RecordIterator == Index->data_end()) 385 return error(instrprof_error::eof); 386 387 // Record the current function name. 388 Record.Name = (*RecordIterator).Name; 389 390 ArrayRef<uint64_t> Data = (*RecordIterator).Data; 391 // Valid data starts with a hash and either a count or the number of counts. 392 if (CurrentOffset + 1 > Data.size()) 393 return error(instrprof_error::malformed); 394 // First we have a function hash. 395 Record.Hash = Data[CurrentOffset++]; 396 // In version 1 we knew the number of counters implicitly, but in newer 397 // versions we store the number of counters next. 398 uint64_t NumCounts = 399 FormatVersion == 1 ? Data.size() - CurrentOffset : Data[CurrentOffset++]; 400 if (CurrentOffset + NumCounts > Data.size()) 401 return error(instrprof_error::malformed); 402 // And finally the counts themselves. 403 Record.Counts = Data.slice(CurrentOffset, NumCounts); 404 405 // If we've exhausted this function's data, increment the record. 406 CurrentOffset += NumCounts; 407 if (CurrentOffset == Data.size()) { 408 ++RecordIterator; 409 CurrentOffset = 0; 410 } 411 412 return success(); 413 } 414
