Home | History | Annotate | Download | only in profile 
      1 /*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\
      2 |*
      3 |*                     The LLVM Compiler Infrastructure
      4 |*
      5 |* This file is distributed under the University of Illinois Open Source
      6 |* License. See LICENSE.TXT for details.
      7 |*
      8 \*===----------------------------------------------------------------------===*/
      9 
     10 #include "InstrProfiling.h"
     11 #include "InstrProfilingInternal.h"
     12 #include <limits.h>
     13 #include <stdio.h>
     14 #include <stdlib.h>
     15 #include <string.h>
     16 #define INSTR_PROF_VALUE_PROF_DATA
     17 #define INSTR_PROF_COMMON_API_IMPL
     18 #include "InstrProfData.inc"
     19 
     20 #define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
     21 #define PROF_OOM_RETURN(Msg)                                                   \
     22   {                                                                            \
     23     PROF_OOM(Msg)                                                              \
     24     return 0;                                                                  \
     25   }
     26 
     27 #if COMPILER_RT_HAS_ATOMICS != 1
     28 COMPILER_RT_VISIBILITY
     29 uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
     30   void *R = *Ptr;
     31   if (R == OldV) {
     32     *Ptr = NewV;
     33     return 1;
     34   }
     35   return 0;
     36 }
     37 #endif
     38 
     39 /* This method is only used in value profiler mock testing.  */
     40 COMPILER_RT_VISIBILITY void
     41 __llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
     42                                    uint32_t ValueKind, uint16_t NumValueSites) {
     43   *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
     44 }
     45 
     46 /* This method is only used in value profiler mock testing.  */
     47 COMPILER_RT_VISIBILITY const __llvm_profile_data *
     48 __llvm_profile_iterate_data(const __llvm_profile_data *Data) {
     49   return Data + 1;
     50 }
     51 
     52 /* This method is only used in value profiler mock testing.  */
     53 COMPILER_RT_VISIBILITY void *
     54 __llvm_get_function_addr(const __llvm_profile_data *Data) {
     55   return Data->FunctionPointer;
     56 }
     57 
     58 /* Allocate an array that holds the pointers to the linked lists of
     59  * value profile counter nodes. The number of element of the array
     60  * is the total number of value profile sites instrumented. Returns
     61  * 0 if allocation fails.
     62  */
     63 
     64 static int allocateValueProfileCounters(__llvm_profile_data *Data) {
     65   uint64_t NumVSites = 0;
     66   uint32_t VKI;
     67   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
     68     NumVSites += Data->NumValueSites[VKI];
     69 
     70   ValueProfNode **Mem =
     71       (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
     72   if (!Mem)
     73     return 0;
     74   if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
     75     free(Mem);
     76     return 0;
     77   }
     78   return 1;
     79 }
     80 
     81 static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
     82   uint64_t NumVSites = 0, I;
     83   uint32_t VKI;
     84   if (!Data->Values)
     85     return;
     86   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
     87     NumVSites += Data->NumValueSites[VKI];
     88   for (I = 0; I < NumVSites; I++) {
     89     ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
     90     while (Node) {
     91       ValueProfNode *Next = Node->Next;
     92       free(Node);
     93       Node = Next;
     94     }
     95   }
     96   free(Data->Values);
     97 }
     98 
     99 COMPILER_RT_VISIBILITY void
    100 __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
    101                                  uint32_t CounterIndex) {
    102 
    103   __llvm_profile_data *PData = (__llvm_profile_data *)Data;
    104   if (!PData)
    105     return;
    106 
    107   if (!PData->Values) {
    108     if (!allocateValueProfileCounters(PData))
    109       return;
    110   }
    111 
    112   ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
    113   ValueProfNode *PrevVNode = NULL;
    114   ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
    115 
    116   uint8_t VDataCount = 0;
    117   while (CurrentVNode) {
    118     if (TargetValue == CurrentVNode->VData.Value) {
    119       CurrentVNode->VData.Count++;
    120       return;
    121     }
    122     PrevVNode = CurrentVNode;
    123     CurrentVNode = CurrentVNode->Next;
    124     ++VDataCount;
    125   }
    126 
    127   if (VDataCount >= UCHAR_MAX)
    128     return;
    129 
    130   CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
    131   if (!CurrentVNode)
    132     return;
    133 
    134   CurrentVNode->VData.Value = TargetValue;
    135   CurrentVNode->VData.Count++;
    136 
    137   uint32_t Success = 0;
    138   if (!ValueCounters[CounterIndex])
    139     Success =
    140         COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
    141   else if (PrevVNode && !PrevVNode->Next)
    142     Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
    143 
    144   if (!Success) {
    145     free(CurrentVNode);
    146     return;
    147   }
    148 }
    149 
    150 /* For multi-threaded programs, while the profile is being dumped, other
    151    threads may still be updating the value profile data and creating new
    152    value entries. To accommadate this, we need to add extra bytes to the
    153    data buffer. The size of the extra space is controlled by an environment
    154    variable. */
    155 static unsigned getVprofExtraBytes() {
    156   const char *ExtraStr =
    157       GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0;
    158   if (!ExtraStr || !ExtraStr[0])
    159     return 1024;
    160   return (unsigned)atoi(ExtraStr);
    161 }
    162 
    163 /* Extract the value profile data info from the runtime. */
    164 #define DEF_VALUE_RECORD(R, NS, V)                                             \
    165   ValueProfRuntimeRecord R;                                                    \
    166   if (initializeValueProfRuntimeRecord(&R, NS, V))                             \
    167     PROF_OOM_RETURN("Failed to write value profile data ");
    168 
    169 #define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R);
    170 
    171 COMPILER_RT_VISIBILITY uint64_t
    172 __llvm_profile_gather_value_data(uint8_t **VDataArray) {
    173   size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
    174   __llvm_profile_data *I;
    175   if (!VDataArray)
    176     PROF_OOM_RETURN("Failed to write value profile data ");
    177 
    178   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
    179   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
    180 
    181   /*
    182    * Compute the total Size of the buffer to hold ValueProfData
    183    * structures for functions with value profile data.
    184    */
    185   for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
    186 
    187     DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
    188 
    189     /* Compute the size of ValueProfData from this runtime record.  */
    190     if (getNumValueKindsRT(&R) != 0)
    191       S += getValueProfDataSizeRT(&R);
    192 
    193     DTOR_VALUE_RECORD(R);
    194   }
    195   /* No value sites or no value profile data is collected. */
    196   if (!S)
    197     return 0;
    198 
    199   Extra = getVprofExtraBytes();
    200   BufferCapacity = S + Extra;
    201   *VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
    202   if (!*VDataArray)
    203     PROF_OOM_RETURN("Failed to write value profile data ");
    204 
    205   ValueProfData *VD = (ValueProfData *)(*VDataArray);
    206   /*
    207    * Extract value profile data and write into ValueProfData structure
    208    * one by one. Note that new value profile data added to any value
    209    * site (from another thread) after the ValueProfRuntimeRecord is
    210    * initialized (when the profile data snapshot is taken) won't be
    211    * collected. This is not a problem as those dropped value will have
    212    * very low taken count.
    213    */
    214   for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
    215     DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
    216     if (getNumValueKindsRT(&R) == 0)
    217       continue;
    218 
    219     /* Record R has taken a snapshot of the VP data at this point. Newly
    220        added VP data for this function will be dropped.  */
    221     /* Check if there is enough space.  */
    222     if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
    223       PROF_ERR("Value profile data is dropped :%s \n",
    224                "Out of buffer space. Use environment "
    225                " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
    226       I->Values = 0;
    227     }
    228 
    229     serializeValueProfDataFromRT(&R, VD);
    230     deallocateValueProfileCounters(I);
    231     I->Values = VD;
    232     RealSize += VD->TotalSize;
    233     VD = (ValueProfData *)((char *)VD + VD->TotalSize);
    234     DTOR_VALUE_RECORD(R);
    235   }
    236 
    237   return RealSize;
    238 }
    239