Home | History | Annotate | Download | only in helgrind
      1 
      2 /*--------------------------------------------------------------------*/
      3 /*--- LibHB: a library for implementing and checking               ---*/
      4 /*--- the happens-before relationship in concurrent programs.      ---*/
      5 /*---                                                 libhb_main.c ---*/
      6 /*--------------------------------------------------------------------*/
      7 
      8 /*
      9    This file is part of LibHB, a library for implementing and checking
     10    the happens-before relationship in concurrent programs.
     11 
     12    Copyright (C) 2008-2015 OpenWorks Ltd
     13       info (at) open-works.co.uk
     14 
     15    This program is free software; you can redistribute it and/or
     16    modify it under the terms of the GNU General Public License as
     17    published by the Free Software Foundation; either version 2 of the
     18    License, or (at your option) any later version.
     19 
     20    This program is distributed in the hope that it will be useful, but
     21    WITHOUT ANY WARRANTY; without even the implied warranty of
     22    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     23    General Public License for more details.
     24 
     25    You should have received a copy of the GNU General Public License
     26    along with this program; if not, write to the Free Software
     27    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
     28    02111-1307, USA.
     29 
     30    The GNU General Public License is contained in the file COPYING.
     31 */
     32 
     33 #include "pub_tool_basics.h"
     34 #include "pub_tool_poolalloc.h"
     35 #include "pub_tool_libcassert.h"
     36 #include "pub_tool_libcbase.h"
     37 #include "pub_tool_libcprint.h"
     38 #include "pub_tool_mallocfree.h"
     39 #include "pub_tool_wordfm.h"
     40 #include "pub_tool_hashtable.h"
     41 #include "pub_tool_xarray.h"
     42 #include "pub_tool_oset.h"
     43 #include "pub_tool_threadstate.h"
     44 #include "pub_tool_aspacemgr.h"
     45 #include "pub_tool_stacktrace.h"
     46 #include "pub_tool_execontext.h"
     47 #include "pub_tool_errormgr.h"
     48 #include "pub_tool_options.h"        // VG_(clo_stats)
     49 #include "hg_basics.h"
     50 #include "hg_wordset.h"
     51 #include "hg_lock_n_thread.h"
     52 #include "hg_errors.h"
     53 
     54 #include "libhb.h"
     55 
     56 
     57 /////////////////////////////////////////////////////////////////
     58 /////////////////////////////////////////////////////////////////
     59 //                                                             //
     60 // Debugging #defines                                          //
     61 //                                                             //
     62 /////////////////////////////////////////////////////////////////
     63 /////////////////////////////////////////////////////////////////
     64 
     65 /* Check the sanity of shadow values in the core memory state
     66    machine.  Change #if 0 to #if 1 to enable this. */
     67 #if 0
     68 #  define CHECK_MSM 1
     69 #else
     70 #  define CHECK_MSM 0
     71 #endif
     72 
     73 
     74 /* Check sanity (reference counts, etc) in the conflicting access
     75    machinery.  Change #if 0 to #if 1 to enable this. */
     76 #if 0
     77 #  define CHECK_CEM 1
     78 #else
     79 #  define CHECK_CEM 0
     80 #endif
     81 
     82 
     83 /* Check sanity in the compressed shadow memory machinery,
     84    particularly in its caching innards.  Unfortunately there's no
     85    almost-zero-cost way to make them selectable at run time.  Hence
     86    set the #if 0 to #if 1 and rebuild if you want them. */
     87 #if 0
     88 #  define CHECK_ZSM 1  /* do sanity-check CacheLine stuff */
     89 #  define inline __attribute__((noinline))
     90    /* probably want to ditch -fomit-frame-pointer too */
     91 #else
     92 #  define CHECK_ZSM 0   /* don't sanity-check CacheLine stuff */
     93 #endif
     94 
     95 
     96 /////////////////////////////////////////////////////////////////
     97 /////////////////////////////////////////////////////////////////
     98 //                                                             //
     99 // data decls: VtsID                                           //
    100 //                                                             //
    101 /////////////////////////////////////////////////////////////////
    102 /////////////////////////////////////////////////////////////////
    103 
    104 /* VtsIDs: Unique small-integer IDs for VTSs.  VtsIDs can't exceed 30
    105    bits, since they have to be packed into the lowest 30 bits of an
    106    SVal. */
    107 typedef  UInt  VtsID;
    108 #define VtsID_INVALID 0xFFFFFFFF
    109 
    110 
    111 
    112 /////////////////////////////////////////////////////////////////
    113 /////////////////////////////////////////////////////////////////
    114 //                                                             //
    115 // data decls: SVal                                            //
    116 //                                                             //
    117 /////////////////////////////////////////////////////////////////
    118 /////////////////////////////////////////////////////////////////
    119 
    120 typedef  ULong  SVal;
    121 
    122 /* This value has special significance to the implementation, and callers
    123    may not store it in the shadow memory. */
    124 #define SVal_INVALID (3ULL << 62)
    125 
    126 /* This is the default value for shadow memory.  Initially the shadow
    127    memory contains no accessible areas and so all reads produce this
    128    value.  TODO: make this caller-defineable. */
    129 #define SVal_NOACCESS (2ULL << 62)
    130 
    131 
    132 
    133 /////////////////////////////////////////////////////////////////
    134 /////////////////////////////////////////////////////////////////
    135 //                                                             //
    136 // data decls: ScalarTS                                        //
    137 //                                                             //
    138 /////////////////////////////////////////////////////////////////
    139 /////////////////////////////////////////////////////////////////
    140 
    141 /* Scalar Timestamp.  We have to store a lot of these, so there is
    142    some effort to make them as small as possible.  Logically they are
    143    a pair, (Thr*, ULong), but that takes 16 bytes on a 64-bit target.
    144    We pack it into 64 bits by representing the Thr* using a ThrID, a
    145    small integer (18 bits), and a 46 bit integer for the timestamp
    146    number.  The 46/18 split is arbitrary, but has the effect that
    147    Helgrind can only handle programs that create 2^18 or fewer threads
    148    over their entire lifetime, and have no more than 2^46 timestamp
    149    ticks (synchronisation operations on the same thread).
    150 
    151    This doesn't seem like much of a limitation.  2^46 ticks is
    152    7.06e+13, and if each tick (optimistically) takes the machine 1000
    153    cycles to process, then the minimum time to process that many ticks
    154    at a clock rate of 5 GHz is 162.9 days.  And that's doing nothing
    155    but VTS ticks, which isn't realistic.
    156 
    157    NB1: SCALARTS_N_THRBITS must be 27 or lower.  The obvious limit is
    158    32 since a ThrID is a UInt.  27 comes from the fact that
    159    'Thr_n_RCEC', which records information about old accesses, packs
    160    in tsw not only a ThrID but also minimum 4+1 other bits (access size
    161    and writeness) in a UInt, hence limiting size to 32-(4+1) == 27.
    162 
    163    NB2: thrid values are issued upwards from 1024, and values less
    164    than that aren't valid.  This isn't per se necessary (any order
    165    will do, so long as they are unique), but it does help ensure they
    166    are less likely to get confused with the various other kinds of
    167    small-integer thread ids drifting around (eg, TId).
    168    So, SCALARTS_N_THRBITS must be 11 or more.
    169    See also NB5.
    170 
    171    NB3: this probably also relies on the fact that Thr's are never
    172    deallocated -- they exist forever.  Hence the 1-1 mapping from
    173    Thr's to thrid values (set up in Thr__new) persists forever.
    174 
    175    NB4: temp_max_sized_VTS is allocated at startup and never freed.
    176    It is a maximum sized VTS, so has (1 << SCALARTS_N_TYMBITS)
    177    ScalarTSs.  So we can't make SCALARTS_N_THRBITS too large without
    178    making the memory use for this go sky-high.  With
    179    SCALARTS_N_THRBITS at 18, it occupies 2MB of memory, which seems
    180    like an OK tradeoff.  If more than 256k threads need to be
    181    supported, we could change SCALARTS_N_THRBITS to 20, which would
    182    facilitate supporting 1 million threads at the cost of 8MB storage
    183    for temp_max_sized_VTS.
    184 
    185    NB5: the conflicting-map mechanism (Thr_n_RCEC, specifically) uses
    186    ThrID == 0 to denote an empty Thr_n_RCEC record.  So ThrID == 0
    187    must never be a valid ThrID.  Given NB2 that's OK.
    188 */
    189 #define SCALARTS_N_THRBITS 18  /* valid range: 11 to 27 inclusive,
    190                                   See NB1 and NB2 above. */
    191 
    192 #define SCALARTS_N_TYMBITS (64 - SCALARTS_N_THRBITS)
    193 typedef
    194    struct {
    195       ThrID thrid : SCALARTS_N_THRBITS;
    196       ULong tym   : SCALARTS_N_TYMBITS;
    197    }
    198    ScalarTS;
    199 
    200 #define ThrID_MAX_VALID ((1 << SCALARTS_N_THRBITS) - 1)
    201 
    202 
    203 
    204 /////////////////////////////////////////////////////////////////
    205 /////////////////////////////////////////////////////////////////
    206 //                                                             //
    207 // data decls: Filter                                          //
    208 //                                                             //
    209 /////////////////////////////////////////////////////////////////
    210 /////////////////////////////////////////////////////////////////
    211 
    212 // baseline: 5, 9
    213 #define FI_LINE_SZB_LOG2  5
    214 #define FI_NUM_LINES_LOG2 10
    215 
    216 #define FI_LINE_SZB       (1 << FI_LINE_SZB_LOG2)
    217 #define FI_NUM_LINES      (1 << FI_NUM_LINES_LOG2)
    218 
    219 #define FI_TAG_MASK        (~(Addr)(FI_LINE_SZB - 1))
    220 #define FI_GET_TAG(_a)     ((_a) & FI_TAG_MASK)
    221 
    222 #define FI_GET_LINENO(_a)  ( ((_a) >> FI_LINE_SZB_LOG2) \
    223                              & (Addr)(FI_NUM_LINES-1) )
    224 
    225 
    226 /* In the lines, each 8 bytes are treated individually, and are mapped
    227    to a UShort.  Regardless of endianness of the underlying machine,
    228    bits 1 and 0 pertain to the lowest address and bits 15 and 14 to
    229    the highest address.
    230 
    231    Of each bit pair, the higher numbered bit is set if a R has been
    232    seen, so the actual layout is:
    233 
    234    15 14             ...  01 00
    235 
    236    R  W  for addr+7  ...  R  W  for addr+0
    237 
    238    So a mask for the R-bits is 0xAAAA and for the W bits is 0x5555.
    239 */
    240 
    241 /* tags are separated from lines.  tags are Addrs and are
    242    the base address of the line. */
    243 typedef
    244    struct {
    245       UShort u16s[FI_LINE_SZB / 8]; /* each UShort covers 8 bytes */
    246    }
    247    FiLine;
    248 
    249 typedef
    250    struct {
    251       Addr   tags[FI_NUM_LINES];
    252       FiLine lines[FI_NUM_LINES];
    253    }
    254    Filter;
    255 
    256 
    257 
    258 /////////////////////////////////////////////////////////////////
    259 /////////////////////////////////////////////////////////////////
    260 //                                                             //
    261 // data decls: Thr, ULong_n_EC                                 //
    262 //                                                             //
    263 /////////////////////////////////////////////////////////////////
    264 /////////////////////////////////////////////////////////////////
    265 
    266 // Records stacks for H1 history mechanism (DRD-style)
    267 typedef
    268    struct { ULong ull; ExeContext* ec; }
    269    ULong_n_EC;
    270 
    271 
    272 /* How many of the above records to collect for each thread?  Older
    273    ones are dumped when we run out of space.  62.5k requires 1MB per
    274    thread, since each ULong_n_EC record is 16 bytes long.  When more
    275    than N_KWs_N_STACKs_PER_THREAD are present, the older half are
    276    deleted to make space.  Hence in the worst case we will be able to
    277    produce a stack at least for the last N_KWs_N_STACKs_PER_THREAD / 2
    278    Kw transitions (segments in this thread).  For the current setting
    279    that gives a guaranteed stack for at least the last 31.25k
    280    segments. */
    281 #define N_KWs_N_STACKs_PER_THREAD 62500
    282 
    283 
    284 struct _Thr {
    285    /* Current VTSs for this thread.  They change as we go along.  viR
    286       is the VTS to be used for reads, viW for writes.  Usually they
    287       are the same, but can differ when we deal with reader-writer
    288       locks.  It is always the case that
    289          VtsID__cmpLEQ(viW,viR) == True
    290       that is, viW must be the same, or lagging behind, viR. */
    291    VtsID viR;
    292    VtsID viW;
    293 
    294    /* Is initially False, and is set to True after the thread really
    295       has done a low-level exit.  When True, we expect to never see
    296       any more memory references done by this thread. */
    297    Bool llexit_done;
    298 
    299    /* Is initially False, and is set to True after the thread has been
    300       joined with (reaped by some other thread).  After this point, we
    301       do not expect to see any uses of .viR or .viW, so it is safe to
    302       set them to VtsID_INVALID. */
    303    Bool joinedwith_done;
    304 
    305    /* A small integer giving a unique identity to this Thr.  See
    306       comments on the definition of ScalarTS for details. */
    307    ThrID thrid : SCALARTS_N_THRBITS;
    308 
    309    /* A filter that removes references for which we believe that
    310       msmcread/msmcwrite will not change the state, nor report a
    311       race. */
    312    Filter* filter;
    313 
    314    /* A pointer back to the top level Thread structure.  There is a
    315       1-1 mapping between Thread and Thr structures -- each Thr points
    316       at its corresponding Thread, and vice versa.  Really, Thr and
    317       Thread should be merged into a single structure. */
    318    Thread* hgthread;
    319 
    320    /* The ULongs (scalar Kws) in this accumulate in strictly
    321       increasing order, without duplicates.  This is important because
    322       we need to be able to find a given scalar Kw in this array
    323       later, by binary search. */
    324    XArray* /* ULong_n_EC */ local_Kws_n_stacks;
    325 };
    326 
    327 
    328 
    329 /////////////////////////////////////////////////////////////////
    330 /////////////////////////////////////////////////////////////////
    331 //                                                             //
    332 // data decls: SO                                              //
    333 //                                                             //
    334 /////////////////////////////////////////////////////////////////
    335 /////////////////////////////////////////////////////////////////
    336 
    337 // (UInt) `echo "Synchronisation object" | md5sum`
    338 #define SO_MAGIC 0x56b3c5b0U
    339 
    340 struct _SO {
    341    struct _SO* admin_prev;
    342    struct _SO* admin_next;
    343    VtsID viR; /* r-clock of sender */
    344    VtsID viW; /* w-clock of sender */
    345    UInt  magic;
    346 };
    347 
    348 
    349 
    350 /////////////////////////////////////////////////////////////////
    351 /////////////////////////////////////////////////////////////////
    352 //                                                             //
    353 // Forward declarations                                        //
    354 //                                                             //
    355 /////////////////////////////////////////////////////////////////
    356 /////////////////////////////////////////////////////////////////
    357 
    358 /* fwds for
    359    Globals needed by other parts of the library.  These are set
    360    once at startup and then never changed. */
    361 static void        (*main_get_stacktrace)( Thr*, Addr*, UWord ) = NULL;
    362 static ExeContext* (*main_get_EC)( Thr* ) = NULL;
    363 
    364 /* misc fn and data fwdses */
    365 static void VtsID__rcinc ( VtsID ii );
    366 static void VtsID__rcdec ( VtsID ii );
    367 
    368 static inline Bool SVal__isC ( SVal s );
    369 static inline VtsID SVal__unC_Rmin ( SVal s );
    370 static inline VtsID SVal__unC_Wmin ( SVal s );
    371 static inline SVal SVal__mkC ( VtsID rmini, VtsID wmini );
    372 static inline void SVal__rcinc ( SVal s );
    373 static inline void SVal__rcdec ( SVal s );
    374 /* SVal in LineZ are used to store various pointers. */
    375 static inline void *SVal2Ptr (SVal s);
    376 static inline SVal Ptr2SVal (void* ptr);
    377 
    378 /* A double linked list of all the SO's. */
    379 SO* admin_SO;
    380 
    381 
    382 
    383 /////////////////////////////////////////////////////////////////
    384 /////////////////////////////////////////////////////////////////
    385 //                                                             //
    386 // SECTION BEGIN compressed shadow memory                      //
    387 //                                                             //
    388 /////////////////////////////////////////////////////////////////
    389 /////////////////////////////////////////////////////////////////
    390 
    391 #ifndef __HB_ZSM_H
    392 #define __HB_ZSM_H
    393 
    394 /* Initialise the library.  Once initialised, it will (or may) call
    395    SVal__rcinc and SVal__rcdec in response to all the calls below, in order to
    396    allow the user to do reference counting on the SVals stored herein.
    397    It is important to understand, however, that due to internal
    398    caching, the reference counts are in general inaccurate, and can be
    399    both above or below the true reference count for an item.  In
    400    particular, the library may indicate that the reference count for
    401    an item is zero, when in fact it is not.
    402 
    403    To make the reference counting exact and therefore non-pointless,
    404    call zsm_flush_cache.  Immediately after it returns, the reference
    405    counts for all items, as deduced by the caller by observing calls
    406    to SVal__rcinc and SVal__rcdec, will be correct, and so any items with a
    407    zero reference count may be freed (or at least considered to be
    408    unreferenced by this library).
    409 */
    410 static void zsm_init ( void );
    411 
    412 static void zsm_sset_range  ( Addr, SizeT, SVal );
    413 static void zsm_sset_range_SMALL ( Addr a, SizeT len, SVal svNew );
    414 static void zsm_scopy_range ( Addr, Addr, SizeT );
    415 static void zsm_flush_cache ( void );
    416 
    417 #endif /* ! __HB_ZSM_H */
    418 
    419 
    420 /* Round a up to the next multiple of N.  N must be a power of 2 */
    421 #define ROUNDUP(a, N)   ((a + N - 1) & ~(N-1))
    422 /* Round a down to the next multiple of N.  N must be a power of 2 */
    423 #define ROUNDDN(a, N)   ((a) & ~(N-1))
    424 
    425 /* True if a belongs in range [start, start + szB[
    426    (i.e. start + szB is excluded). */
    427 static inline Bool address_in_range (Addr a, Addr start,  SizeT szB)
    428 {
    429    /* Checking start <= a && a < start + szB.
    430       As start and a are unsigned addresses, the condition can
    431       be simplified. */
    432    if (CHECK_ZSM)
    433       tl_assert ((a - start < szB)
    434                  == (start <= a
    435                      &&       a < start + szB));
    436    return a - start < szB;
    437 }
    438 
    439 /* ------ CacheLine ------ */
    440 
    441 #define N_LINE_BITS      6 /* must be >= 3 */
    442 #define N_LINE_ARANGE    (1 << N_LINE_BITS)
    443 #define N_LINE_TREES     (N_LINE_ARANGE >> 3)
    444 
    445 typedef
    446    struct {
    447       UShort descrs[N_LINE_TREES];
    448       SVal   svals[N_LINE_ARANGE]; // == N_LINE_TREES * 8
    449    }
    450    CacheLine;
    451 
    452 #define TREE_DESCR_16_0 (1<<0)
    453 #define TREE_DESCR_32_0 (1<<1)
    454 #define TREE_DESCR_16_1 (1<<2)
    455 #define TREE_DESCR_64   (1<<3)
    456 #define TREE_DESCR_16_2 (1<<4)
    457 #define TREE_DESCR_32_1 (1<<5)
    458 #define TREE_DESCR_16_3 (1<<6)
    459 #define TREE_DESCR_8_0  (1<<7)
    460 #define TREE_DESCR_8_1  (1<<8)
    461 #define TREE_DESCR_8_2  (1<<9)
    462 #define TREE_DESCR_8_3  (1<<10)
    463 #define TREE_DESCR_8_4  (1<<11)
    464 #define TREE_DESCR_8_5  (1<<12)
    465 #define TREE_DESCR_8_6  (1<<13)
    466 #define TREE_DESCR_8_7  (1<<14)
    467 #define TREE_DESCR_DTY  (1<<15)
    468 
    469 typedef
    470    struct {
    471       SVal  dict[4]; /* can represent up to 4 diff values in the line */
    472       UChar ix2s[N_LINE_ARANGE/4]; /* array of N_LINE_ARANGE 2-bit
    473                                       dict indexes */
    474       /* if dict[0] == SVal_INVALID then dict[1] is a pointer to the
    475          LineF to use, and dict[2..] are also SVal_INVALID. */
    476    }
    477    LineZ; /* compressed rep for a cache line */
    478 
    479 /* LineZ.dict[1] is used to store various pointers:
    480    * In the first lineZ of a free SecMap, it points to the next free SecMap.
    481    * In a lineZ for which we need to use a lineF, it points to the lineF. */
    482 
    483 
    484 typedef
    485    struct {
    486       SVal w64s[N_LINE_ARANGE];
    487    }
    488    LineF; /* full rep for a cache line */
    489 
    490 /* We use a pool allocator for LineF, as LineF is relatively small,
    491    and we will often alloc/release such lines. */
    492 static PoolAlloc* LineF_pool_allocator;
    493 
    494 /* SVal in a lineZ are used to store various pointers.
    495    Below are conversion functions to support that. */
    496 static inline LineF *LineF_Ptr (LineZ *lineZ)
    497 {
    498    tl_assert(lineZ->dict[0] == SVal_INVALID);
    499    return SVal2Ptr (lineZ->dict[1]);
    500 }
    501 
    502 /* Shadow memory.
    503    Primary map is a WordFM Addr SecMap*.
    504    SecMaps cover some page-size-ish section of address space and hold
    505      a compressed representation.
    506    CacheLine-sized chunks of SecMaps are copied into a Cache, being
    507    decompressed when moved into the cache and recompressed on the
    508    way out.  Because of this, the cache must operate as a writeback
    509    cache, not a writethrough one.
    510 
    511    Each SecMap must hold a power-of-2 number of CacheLines.  Hence
    512    N_SECMAP_BITS must >= N_LINE_BITS.
    513 */
    514 #define N_SECMAP_BITS   13
    515 #define N_SECMAP_ARANGE (1 << N_SECMAP_BITS)
    516 
    517 // # CacheLines held by a SecMap
    518 #define N_SECMAP_ZLINES (N_SECMAP_ARANGE / N_LINE_ARANGE)
    519 
    520 /* The data in the SecMap is held in the array of LineZs.  Each LineZ
    521    either carries the required data directly, in a compressed
    522    representation, or it holds (in .dict[1]) a pointer to a LineF
    523    that holds the full representation.
    524 
    525    As each in-use LineF is referred to by exactly one LineZ,
    526    the number of .linesZ[] that refer to a lineF should equal
    527    the number of used lineF.
    528 
    529    RC obligations: the RCs presented to the user include exactly
    530    the values in:
    531    * direct Z reps, that is, ones for which .dict[0] != SVal_INVALID
    532    * F reps that are in use
    533 
    534    Hence the following actions at the following transitions are required:
    535 
    536    F rep: alloc'd       -> freed                -- rcdec_LineF
    537    F rep:               -> alloc'd              -- rcinc_LineF
    538    Z rep: .dict[0] from other to SVal_INVALID   -- rcdec_LineZ
    539    Z rep: .dict[0] from SVal_INVALID to other   -- rcinc_LineZ
    540 */
    541 
    542 typedef
    543    struct {
    544       UInt   magic;
    545       LineZ  linesZ[N_SECMAP_ZLINES];
    546    }
    547    SecMap;
    548 
    549 #define SecMap_MAGIC   0x571e58cbU
    550 
    551 // (UInt) `echo "Free SecMap" | md5sum`
    552 #define SecMap_free_MAGIC 0x5a977f30U
    553 
    554 __attribute__((unused))
    555 static inline Bool is_sane_SecMap ( SecMap* sm ) {
    556    return sm != NULL && sm->magic == SecMap_MAGIC;
    557 }
    558 
    559 /* ------ Cache ------ */
    560 
    561 #define N_WAY_BITS 16
    562 #define N_WAY_NENT (1 << N_WAY_BITS)
    563 
    564 /* Each tag is the address of the associated CacheLine, rounded down
    565    to a CacheLine address boundary.  A CacheLine size must be a power
    566    of 2 and must be 8 or more.  Hence an easy way to initialise the
    567    cache so it is empty is to set all the tag values to any value % 8
    568    != 0, eg 1.  This means all queries in the cache initially miss.
    569    It does however require us to detect and not writeback, any line
    570    with a bogus tag. */
    571 typedef
    572    struct {
    573       CacheLine lyns0[N_WAY_NENT];
    574       Addr      tags0[N_WAY_NENT];
    575    }
    576    Cache;
    577 
    578 static inline Bool is_valid_scache_tag ( Addr tag ) {
    579    /* a valid tag should be naturally aligned to the start of
    580       a CacheLine. */
    581    return 0 == (tag & (N_LINE_ARANGE - 1));
    582 }
    583 
    584 
    585 /* --------- Primary data structures --------- */
    586 
    587 /* Shadow memory primary map */
    588 static WordFM* map_shmem = NULL; /* WordFM Addr SecMap* */
    589 static Cache   cache_shmem;
    590 
    591 
    592 static UWord stats__secmaps_search       = 0; // # SM finds
    593 static UWord stats__secmaps_search_slow  = 0; // # SM lookupFMs
    594 static UWord stats__secmaps_allocd       = 0; // # SecMaps issued
    595 static UWord stats__secmaps_in_map_shmem = 0; // # SecMaps 'live'
    596 static UWord stats__secmaps_scanGC       = 0; // # nr of scan GC done.
    597 static UWord stats__secmaps_scanGCed     = 0; // # SecMaps GC-ed via scan
    598 static UWord stats__secmaps_ssetGCed     = 0; // # SecMaps GC-ed via setnoaccess
    599 static UWord stats__secmap_ga_space_covered = 0; // # ga bytes covered
    600 static UWord stats__secmap_linesZ_allocd = 0; // # LineZ's issued
    601 static UWord stats__secmap_linesZ_bytes  = 0; // .. using this much storage
    602 static UWord stats__cache_Z_fetches      = 0; // # Z lines fetched
    603 static UWord stats__cache_Z_wbacks       = 0; // # Z lines written back
    604 static UWord stats__cache_F_fetches      = 0; // # F lines fetched
    605 static UWord stats__cache_F_wbacks       = 0; // # F lines written back
    606 static UWord stats__cache_flushes_invals = 0; // # cache flushes and invals
    607 static UWord stats__cache_totrefs        = 0; // # total accesses
    608 static UWord stats__cache_totmisses      = 0; // # misses
    609 static ULong stats__cache_make_New_arange = 0; // total arange made New
    610 static ULong stats__cache_make_New_inZrep = 0; // arange New'd on Z reps
    611 static UWord stats__cline_normalises     = 0; // # calls to cacheline_normalise
    612 static UWord stats__cline_cread64s       = 0; // # calls to s_m_read64
    613 static UWord stats__cline_cread32s       = 0; // # calls to s_m_read32
    614 static UWord stats__cline_cread16s       = 0; // # calls to s_m_read16
    615 static UWord stats__cline_cread08s       = 0; // # calls to s_m_read8
    616 static UWord stats__cline_cwrite64s      = 0; // # calls to s_m_write64
    617 static UWord stats__cline_cwrite32s      = 0; // # calls to s_m_write32
    618 static UWord stats__cline_cwrite16s      = 0; // # calls to s_m_write16
    619 static UWord stats__cline_cwrite08s      = 0; // # calls to s_m_write8
    620 static UWord stats__cline_sread08s       = 0; // # calls to s_m_set8
    621 static UWord stats__cline_swrite08s      = 0; // # calls to s_m_get8
    622 static UWord stats__cline_swrite16s      = 0; // # calls to s_m_get8
    623 static UWord stats__cline_swrite32s      = 0; // # calls to s_m_get8
    624 static UWord stats__cline_swrite64s      = 0; // # calls to s_m_get8
    625 static UWord stats__cline_scopy08s       = 0; // # calls to s_m_copy8
    626 static UWord stats__cline_64to32splits   = 0; // # 64-bit accesses split
    627 static UWord stats__cline_32to16splits   = 0; // # 32-bit accesses split
    628 static UWord stats__cline_16to8splits    = 0; // # 16-bit accesses split
    629 static UWord stats__cline_64to32pulldown = 0; // # calls to pulldown_to_32
    630 static UWord stats__cline_32to16pulldown = 0; // # calls to pulldown_to_16
    631 static UWord stats__cline_16to8pulldown  = 0; // # calls to pulldown_to_8
    632 static UWord stats__vts__tick            = 0; // # calls to VTS__tick
    633 static UWord stats__vts__join            = 0; // # calls to VTS__join
    634 static UWord stats__vts__cmpLEQ          = 0; // # calls to VTS__cmpLEQ
    635 static UWord stats__vts__cmp_structural  = 0; // # calls to VTS__cmp_structural
    636 static UWord stats__vts_tab_GC           = 0; // # nr of vts_tab GC
    637 static UWord stats__vts_pruning          = 0; // # nr of vts pruning
    638 
    639 // # calls to VTS__cmp_structural w/ slow case
    640 static UWord stats__vts__cmp_structural_slow = 0;
    641 
    642 // # calls to VTS__indexAt_SLOW
    643 static UWord stats__vts__indexat_slow = 0;
    644 
    645 // # calls to vts_set__find__or__clone_and_add
    646 static UWord stats__vts_set__focaa    = 0;
    647 
    648 // # calls to vts_set__find__or__clone_and_add that lead to an
    649 // allocation
    650 static UWord stats__vts_set__focaa_a  = 0;
    651 
    652 
    653 static inline Addr shmem__round_to_SecMap_base ( Addr a ) {
    654    return a & ~(N_SECMAP_ARANGE - 1);
    655 }
    656 static inline UWord shmem__get_SecMap_offset ( Addr a ) {
    657    return a & (N_SECMAP_ARANGE - 1);
    658 }
    659 
    660 
    661 /*----------------------------------------------------------------*/
    662 /*--- map_shmem :: WordFM Addr SecMap                          ---*/
    663 /*--- shadow memory (low level handlers) (shmem__* fns)        ---*/
    664 /*----------------------------------------------------------------*/
    665 
    666 /*--------------- SecMap allocation --------------- */
    667 
    668 static HChar* shmem__bigchunk_next = NULL;
    669 static HChar* shmem__bigchunk_end1 = NULL;
    670 
    671 static void* shmem__bigchunk_alloc ( SizeT n )
    672 {
    673    const SizeT sHMEM__BIGCHUNK_SIZE = 4096 * 256 * 4;
    674    tl_assert(n > 0);
    675    n = VG_ROUNDUP(n, 16);
    676    tl_assert(shmem__bigchunk_next <= shmem__bigchunk_end1);
    677    tl_assert(shmem__bigchunk_end1 - shmem__bigchunk_next
    678              <= (SSizeT)sHMEM__BIGCHUNK_SIZE);
    679    if (shmem__bigchunk_next + n > shmem__bigchunk_end1) {
    680       if (0)
    681       VG_(printf)("XXXXX bigchunk: abandoning %d bytes\n",
    682                   (Int)(shmem__bigchunk_end1 - shmem__bigchunk_next));
    683       shmem__bigchunk_next = VG_(am_shadow_alloc)( sHMEM__BIGCHUNK_SIZE );
    684       if (shmem__bigchunk_next == NULL)
    685          VG_(out_of_memory_NORETURN)(
    686             "helgrind:shmem__bigchunk_alloc", sHMEM__BIGCHUNK_SIZE );
    687       shmem__bigchunk_end1 = shmem__bigchunk_next + sHMEM__BIGCHUNK_SIZE;
    688    }
    689    tl_assert(shmem__bigchunk_next);
    690    tl_assert( 0 == (((Addr)shmem__bigchunk_next) & (16-1)) );
    691    tl_assert(shmem__bigchunk_next + n <= shmem__bigchunk_end1);
    692    shmem__bigchunk_next += n;
    693    return shmem__bigchunk_next - n;
    694 }
    695 
    696 /* SecMap changed to be fully SVal_NOACCESS are inserted in a list of
    697    recycled SecMap. When a new SecMap is needed, a recycled SecMap
    698    will be used in preference to allocating a new SecMap. */
    699 /* We make a linked list of SecMap. The first LineZ is re-used to
    700    implement the linked list. */
    701 /* Returns the SecMap following sm in the free list.
    702    NULL if sm is the last SecMap. sm must be on the free list. */
    703 static inline SecMap *SecMap_freelist_next ( SecMap* sm )
    704 {
    705    tl_assert (sm);
    706    tl_assert (sm->magic == SecMap_free_MAGIC);
    707    return SVal2Ptr (sm->linesZ[0].dict[1]);
    708 }
    709 static inline void set_SecMap_freelist_next ( SecMap* sm, SecMap* next )
    710 {
    711    tl_assert (sm);
    712    tl_assert (sm->magic == SecMap_free_MAGIC);
    713    tl_assert (next == NULL || next->magic == SecMap_free_MAGIC);
    714    sm->linesZ[0].dict[1] = Ptr2SVal (next);
    715 }
    716 
    717 static SecMap *SecMap_freelist = NULL;
    718 static UWord SecMap_freelist_length(void)
    719 {
    720    SecMap *sm;
    721    UWord n = 0;
    722 
    723    sm = SecMap_freelist;
    724    while (sm) {
    725      n++;
    726      sm = SecMap_freelist_next (sm);
    727    }
    728    return n;
    729 }
    730 
    731 static void push_SecMap_on_freelist(SecMap* sm)
    732 {
    733    if (0) VG_(message)(Vg_DebugMsg, "%p push\n", sm);
    734    sm->magic = SecMap_free_MAGIC;
    735    set_SecMap_freelist_next(sm, SecMap_freelist);
    736    SecMap_freelist = sm;
    737 }
    738 /* Returns a free SecMap if there is one.
    739    Otherwise, returns NULL. */
    740 static SecMap *pop_SecMap_from_freelist(void)
    741 {
    742    SecMap *sm;
    743 
    744    sm = SecMap_freelist;
    745    if (sm) {
    746       tl_assert (sm->magic == SecMap_free_MAGIC);
    747       SecMap_freelist = SecMap_freelist_next (sm);
    748       if (0) VG_(message)(Vg_DebugMsg, "%p pop\n", sm);
    749    }
    750    return sm;
    751 }
    752 
    753 static SecMap* shmem__alloc_or_recycle_SecMap ( void )
    754 {
    755    Word    i, j;
    756    SecMap* sm = pop_SecMap_from_freelist();
    757 
    758    if (!sm) {
    759       sm = shmem__bigchunk_alloc( sizeof(SecMap) );
    760       stats__secmaps_allocd++;
    761       stats__secmap_ga_space_covered += N_SECMAP_ARANGE;
    762       stats__secmap_linesZ_allocd += N_SECMAP_ZLINES;
    763       stats__secmap_linesZ_bytes += N_SECMAP_ZLINES * sizeof(LineZ);
    764    }
    765    if (0) VG_(printf)("alloc_SecMap %p\n",sm);
    766    tl_assert(sm);
    767    sm->magic = SecMap_MAGIC;
    768    for (i = 0; i < N_SECMAP_ZLINES; i++) {
    769       sm->linesZ[i].dict[0] = SVal_NOACCESS;
    770       sm->linesZ[i].dict[1] = SVal_INVALID;
    771       sm->linesZ[i].dict[2] = SVal_INVALID;
    772       sm->linesZ[i].dict[3] = SVal_INVALID;
    773       for (j = 0; j < N_LINE_ARANGE/4; j++)
    774          sm->linesZ[i].ix2s[j] = 0; /* all reference dict[0] */
    775    }
    776    return sm;
    777 }
    778 
    779 typedef struct { Addr gaKey; SecMap* sm; } SMCacheEnt;
    780 static SMCacheEnt smCache[3] = { {1,NULL}, {1,NULL}, {1,NULL} };
    781 
    782 static SecMap* shmem__find_SecMap ( Addr ga )
    783 {
    784    SecMap* sm    = NULL;
    785    Addr    gaKey = shmem__round_to_SecMap_base(ga);
    786    // Cache
    787    stats__secmaps_search++;
    788    if (LIKELY(gaKey == smCache[0].gaKey))
    789       return smCache[0].sm;
    790    if (LIKELY(gaKey == smCache[1].gaKey)) {
    791       SMCacheEnt tmp = smCache[0];
    792       smCache[0] = smCache[1];
    793       smCache[1] = tmp;
    794       return smCache[0].sm;
    795    }
    796    if (gaKey == smCache[2].gaKey) {
    797       SMCacheEnt tmp = smCache[1];
    798       smCache[1] = smCache[2];
    799       smCache[2] = tmp;
    800       return smCache[1].sm;
    801    }
    802    // end Cache
    803    stats__secmaps_search_slow++;
    804    if (VG_(lookupFM)( map_shmem,
    805                       NULL/*keyP*/, (UWord*)&sm, (UWord)gaKey )) {
    806       tl_assert(sm != NULL);
    807       smCache[2] = smCache[1];
    808       smCache[1] = smCache[0];
    809       smCache[0].gaKey = gaKey;
    810       smCache[0].sm    = sm;
    811    } else {
    812       tl_assert(sm == NULL);
    813    }
    814    return sm;
    815 }
    816 
    817 /* Scan the SecMap and count the SecMap that can be GC-ed.
    818    If really, really does the GC of the SecMap. */
    819 /* NOT TO BE CALLED FROM WITHIN libzsm. */
    820 static UWord next_SecMap_GC_at = 1000;
    821 __attribute__((noinline))
    822 static UWord shmem__SecMap_do_GC(Bool really)
    823 {
    824    UWord secmapW = 0;
    825    Addr  gaKey;
    826    UWord examined = 0;
    827    UWord ok_GCed = 0;
    828 
    829    /* First invalidate the smCache */
    830    smCache[0].gaKey = 1;
    831    smCache[1].gaKey = 1;
    832    smCache[2].gaKey = 1;
    833    STATIC_ASSERT (3 == sizeof(smCache)/sizeof(smCache[0]));
    834 
    835    VG_(initIterFM)( map_shmem );
    836    while (VG_(nextIterFM)( map_shmem, &gaKey, &secmapW )) {
    837       UWord   i;
    838       UWord   j;
    839       UWord   n_linesF = 0;
    840       SecMap* sm = (SecMap*)secmapW;
    841       tl_assert(sm->magic == SecMap_MAGIC);
    842       Bool ok_to_GC = True;
    843 
    844       examined++;
    845 
    846       /* Deal with the LineZs and the possible LineF of a LineZ. */
    847       for (i = 0; i < N_SECMAP_ZLINES && ok_to_GC; i++) {
    848          LineZ* lineZ = &sm->linesZ[i];
    849          if (lineZ->dict[0] != SVal_INVALID) {
    850             ok_to_GC = lineZ->dict[0] == SVal_NOACCESS
    851                && !SVal__isC (lineZ->dict[1])
    852                && !SVal__isC (lineZ->dict[2])
    853                && !SVal__isC (lineZ->dict[3]);
    854          } else {
    855             LineF *lineF = LineF_Ptr(lineZ);
    856             n_linesF++;
    857             for (j = 0; j < N_LINE_ARANGE && ok_to_GC; j++)
    858                ok_to_GC = lineF->w64s[j] == SVal_NOACCESS;
    859          }
    860       }
    861       if (ok_to_GC)
    862          ok_GCed++;
    863       if (ok_to_GC && really) {
    864         SecMap *fm_sm;
    865         Addr fm_gaKey;
    866         /* We cannot remove a SecMap from map_shmem while iterating.
    867            So, stop iteration, remove from map_shmem, recreate the iteration
    868            on the next SecMap. */
    869         VG_(doneIterFM) ( map_shmem );
    870         /* No need to rcdec linesZ or linesF, these are all SVal_NOACCESS.
    871            We just need to free the lineF referenced by the linesZ. */
    872         if (n_linesF > 0) {
    873            for (i = 0; i < N_SECMAP_ZLINES && n_linesF > 0; i++) {
    874               LineZ* lineZ = &sm->linesZ[i];
    875               if (lineZ->dict[0] == SVal_INVALID) {
    876                  VG_(freeEltPA)( LineF_pool_allocator, LineF_Ptr(lineZ) );
    877                  n_linesF--;
    878               }
    879            }
    880         }
    881         if (!VG_(delFromFM)(map_shmem, &fm_gaKey, (UWord*)&fm_sm, gaKey))
    882           tl_assert (0);
    883         stats__secmaps_in_map_shmem--;
    884         tl_assert (gaKey == fm_gaKey);
    885         tl_assert (sm == fm_sm);
    886         stats__secmaps_scanGCed++;
    887         push_SecMap_on_freelist (sm);
    888         VG_(initIterAtFM) (map_shmem, gaKey + N_SECMAP_ARANGE);
    889       }
    890    }
    891    VG_(doneIterFM)( map_shmem );
    892 
    893    if (really) {
    894       stats__secmaps_scanGC++;
    895       /* Next GC when we approach the max allocated */
    896       next_SecMap_GC_at = stats__secmaps_allocd - 1000;
    897       /* Unless we GCed less than 10%. We then allow to alloc 10%
    898          more before GCing. This avoids doing a lot of costly GC
    899          for the worst case : the 'growing phase' of an application
    900          that allocates a lot of memory.
    901          Worst can can be reproduced e.g. by
    902              perf/memrw -t 30000000 -b 1000 -r 1 -l 1
    903          that allocates around 30Gb of memory. */
    904       if (ok_GCed < stats__secmaps_allocd/10)
    905          next_SecMap_GC_at = stats__secmaps_allocd + stats__secmaps_allocd/10;
    906 
    907    }
    908 
    909    if (VG_(clo_stats) && really) {
    910       VG_(message)(Vg_DebugMsg,
    911                   "libhb: SecMap GC: #%lu scanned %lu, GCed %lu,"
    912                    " next GC at %lu\n",
    913                    stats__secmaps_scanGC, examined, ok_GCed,
    914                    next_SecMap_GC_at);
    915    }
    916 
    917    return ok_GCed;
    918 }
    919 
    920 static SecMap* shmem__find_or_alloc_SecMap ( Addr ga )
    921 {
    922    SecMap* sm = shmem__find_SecMap ( ga );
    923    if (LIKELY(sm)) {
    924       if (CHECK_ZSM) tl_assert(is_sane_SecMap(sm));
    925       return sm;
    926    } else {
    927       /* create a new one */
    928       Addr gaKey = shmem__round_to_SecMap_base(ga);
    929       sm = shmem__alloc_or_recycle_SecMap();
    930       tl_assert(sm);
    931       VG_(addToFM)( map_shmem, (UWord)gaKey, (UWord)sm );
    932       stats__secmaps_in_map_shmem++;
    933       if (CHECK_ZSM) tl_assert(is_sane_SecMap(sm));
    934       return sm;
    935    }
    936 }
    937 
    938 /* Returns the nr of linesF which are in use. Note: this is scanning
    939    the secmap wordFM. So, this is to be used for statistics only. */
    940 __attribute__((noinline))
    941 static UWord shmem__SecMap_used_linesF(void)
    942 {
    943    UWord secmapW = 0;
    944    Addr  gaKey;
    945    UWord inUse = 0;
    946 
    947    VG_(initIterFM)( map_shmem );
    948    while (VG_(nextIterFM)( map_shmem, &gaKey, &secmapW )) {
    949       UWord   i;
    950       SecMap* sm = (SecMap*)secmapW;
    951       tl_assert(sm->magic == SecMap_MAGIC);
    952 
    953       for (i = 0; i < N_SECMAP_ZLINES; i++) {
    954          LineZ* lineZ = &sm->linesZ[i];
    955          if (lineZ->dict[0] == SVal_INVALID)
    956             inUse++;
    957       }
    958    }
    959    VG_(doneIterFM)( map_shmem );
    960 
    961    return inUse;
    962 }
    963 
    964 /* ------------ LineF and LineZ related ------------ */
    965 
    966 static void rcinc_LineF ( LineF* lineF ) {
    967    UWord i;
    968    for (i = 0; i < N_LINE_ARANGE; i++)
    969       SVal__rcinc(lineF->w64s[i]);
    970 }
    971 
    972 static void rcdec_LineF ( LineF* lineF ) {
    973    UWord i;
    974    for (i = 0; i < N_LINE_ARANGE; i++)
    975       SVal__rcdec(lineF->w64s[i]);
    976 }
    977 
    978 static void rcinc_LineZ ( LineZ* lineZ ) {
    979    tl_assert(lineZ->dict[0] != SVal_INVALID);
    980    SVal__rcinc(lineZ->dict[0]);
    981    if (lineZ->dict[1] != SVal_INVALID) SVal__rcinc(lineZ->dict[1]);
    982    if (lineZ->dict[2] != SVal_INVALID) SVal__rcinc(lineZ->dict[2]);
    983    if (lineZ->dict[3] != SVal_INVALID) SVal__rcinc(lineZ->dict[3]);
    984 }
    985 
    986 static void rcdec_LineZ ( LineZ* lineZ ) {
    987    tl_assert(lineZ->dict[0] != SVal_INVALID);
    988    SVal__rcdec(lineZ->dict[0]);
    989    if (lineZ->dict[1] != SVal_INVALID) SVal__rcdec(lineZ->dict[1]);
    990    if (lineZ->dict[2] != SVal_INVALID) SVal__rcdec(lineZ->dict[2]);
    991    if (lineZ->dict[3] != SVal_INVALID) SVal__rcdec(lineZ->dict[3]);
    992 }
    993 
    994 inline
    995 static void write_twobit_array ( UChar* arr, UWord ix, UWord b2 ) {
    996    Word bix, shft, mask, prep;
    997    tl_assert(ix >= 0);
    998    bix  = ix >> 2;
    999    shft = 2 * (ix & 3); /* 0, 2, 4 or 6 */
   1000    mask = 3 << shft;
   1001    prep = b2 << shft;
   1002    arr[bix] = (arr[bix] & ~mask) | prep;
   1003 }
   1004 
   1005 inline
   1006 static UWord read_twobit_array ( UChar* arr, UWord ix ) {
   1007    Word bix, shft;
   1008    tl_assert(ix >= 0);
   1009    bix  = ix >> 2;
   1010    shft = 2 * (ix & 3); /* 0, 2, 4 or 6 */
   1011    return (arr[bix] >> shft) & 3;
   1012 }
   1013 
   1014 /* We cache one free lineF, to avoid pool allocator calls.
   1015    Measurement on firefox has shown that this avoids more than 90%
   1016    of the PA calls. */
   1017 static LineF *free_lineF = NULL;
   1018 
   1019 /* Allocates a lineF for LineZ. Sets lineZ in a state indicating
   1020    lineF has to be used. */
   1021 static inline LineF *alloc_LineF_for_Z (LineZ *lineZ)
   1022 {
   1023    LineF *lineF;
   1024 
   1025    tl_assert(lineZ->dict[0] == SVal_INVALID);
   1026 
   1027    if (LIKELY(free_lineF)) {
   1028       lineF = free_lineF;
   1029       free_lineF = NULL;
   1030    } else {
   1031       lineF = VG_(allocEltPA) ( LineF_pool_allocator );
   1032    }
   1033    lineZ->dict[0] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID;
   1034    lineZ->dict[1] = Ptr2SVal (lineF);
   1035 
   1036    return lineF;
   1037 }
   1038 
   1039 /* rcdec the LineF of lineZ, frees the lineF, and sets lineZ
   1040    back to its initial state SVal_NOACCESS (i.e. ready to be
   1041    read or written just after SecMap allocation). */
   1042 static inline void clear_LineF_of_Z (LineZ *lineZ)
   1043 {
   1044    LineF *lineF = LineF_Ptr(lineZ);
   1045 
   1046    rcdec_LineF(lineF);
   1047    if (UNLIKELY(free_lineF)) {
   1048       VG_(freeEltPA)( LineF_pool_allocator, lineF );
   1049    } else {
   1050       free_lineF = lineF;
   1051    }
   1052    lineZ->dict[0] = SVal_NOACCESS;
   1053    lineZ->dict[1] = SVal_INVALID;
   1054 }
   1055 
   1056 /* Given address 'tag', find either the Z or F line containing relevant
   1057    data, so it can be read into the cache.
   1058 */
   1059 static void find_ZF_for_reading ( /*OUT*/LineZ** zp,
   1060                                   /*OUT*/LineF** fp, Addr tag ) {
   1061    LineZ* lineZ;
   1062    LineF* lineF;
   1063    UWord   zix;
   1064    SecMap* sm    = shmem__find_or_alloc_SecMap(tag);
   1065    UWord   smoff = shmem__get_SecMap_offset(tag);
   1066    /* since smoff is derived from a valid tag, it should be
   1067       cacheline-aligned. */
   1068    tl_assert(0 == (smoff & (N_LINE_ARANGE - 1)));
   1069    zix = smoff >> N_LINE_BITS;
   1070    tl_assert(zix < N_SECMAP_ZLINES);
   1071    lineZ = &sm->linesZ[zix];
   1072    lineF = NULL;
   1073    if (lineZ->dict[0] == SVal_INVALID) {
   1074       lineF = LineF_Ptr (lineZ);
   1075       lineZ = NULL;
   1076    }
   1077    *zp = lineZ;
   1078    *fp = lineF;
   1079 }
   1080 
   1081 /* Given address 'tag', return the relevant SecMap and the index of
   1082    the LineZ within it, in the expectation that the line is to be
   1083    overwritten.  Regardless of whether 'tag' is currently associated
   1084    with a Z or F representation, to rcdec on the current
   1085    representation, in recognition of the fact that the contents are
   1086    just about to be overwritten. */
   1087 static __attribute__((noinline))
   1088 void find_Z_for_writing ( /*OUT*/SecMap** smp,
   1089                           /*OUT*/Word* zixp,
   1090                           Addr tag ) {
   1091    LineZ* lineZ;
   1092    UWord   zix;
   1093    SecMap* sm    = shmem__find_or_alloc_SecMap(tag);
   1094    UWord   smoff = shmem__get_SecMap_offset(tag);
   1095    /* since smoff is derived from a valid tag, it should be
   1096       cacheline-aligned. */
   1097    tl_assert(0 == (smoff & (N_LINE_ARANGE - 1)));
   1098    zix = smoff >> N_LINE_BITS;
   1099    tl_assert(zix < N_SECMAP_ZLINES);
   1100    lineZ = &sm->linesZ[zix];
   1101    /* re RCs, we are rcdec_LineZ/clear_LineF_of_Z this LineZ so that new data
   1102       can be parked in it.  Hence have to rcdec it accordingly. */
   1103    /* If lineZ has an associated lineF, free it up. */
   1104    if (lineZ->dict[0] == SVal_INVALID)
   1105       clear_LineF_of_Z(lineZ);
   1106    else
   1107       rcdec_LineZ(lineZ);
   1108    *smp  = sm;
   1109    *zixp = zix;
   1110 }
   1111 
   1112 /* ------------ CacheLine and implicit-tree related ------------ */
   1113 
   1114 __attribute__((unused))
   1115 static void pp_CacheLine ( CacheLine* cl ) {
   1116    Word i;
   1117    if (!cl) {
   1118       VG_(printf)("%s","pp_CacheLine(NULL)\n");
   1119       return;
   1120    }
   1121    for (i = 0; i < N_LINE_TREES; i++)
   1122       VG_(printf)("   descr: %04lx\n", (UWord)cl->descrs[i]);
   1123    for (i = 0; i < N_LINE_ARANGE; i++)
   1124       VG_(printf)("    sval: %08lx\n", (UWord)cl->svals[i]);
   1125 }
   1126 
   1127 static UChar descr_to_validbits ( UShort descr )
   1128 {
   1129    /* a.k.a Party Time for gcc's constant folder */
   1130 #  define DESCR(b8_7, b8_6, b8_5, b8_4, b8_3, b8_2, b8_1, b8_0, \
   1131                 b16_3, b32_1, b16_2, b64, b16_1, b32_0, b16_0)  \
   1132              ( (UShort) ( ( (b8_7)  << 14) | ( (b8_6)  << 13) | \
   1133                           ( (b8_5)  << 12) | ( (b8_4)  << 11) | \
   1134                           ( (b8_3)  << 10) | ( (b8_2)  << 9)  | \
   1135                           ( (b8_1)  << 8)  | ( (b8_0)  << 7)  | \
   1136                           ( (b16_3) << 6)  | ( (b32_1) << 5)  | \
   1137                           ( (b16_2) << 4)  | ( (b64)   << 3)  | \
   1138                           ( (b16_1) << 2)  | ( (b32_0) << 1)  | \
   1139                           ( (b16_0) << 0) ) )
   1140 
   1141 #  define BYTE(bit7, bit6, bit5, bit4, bit3, bit2, bit1, bit0) \
   1142              ( (UChar) ( ( (bit7) << 7) | ( (bit6) << 6) | \
   1143                          ( (bit5) << 5) | ( (bit4) << 4) | \
   1144                          ( (bit3) << 3) | ( (bit2) << 2) | \
   1145                          ( (bit1) << 1) | ( (bit0) << 0) ) )
   1146 
   1147    /* these should all get folded out at compile time */
   1148    tl_assert(DESCR(1,0,0,0,0,0,0,0, 0,0,0, 0, 0,0,0) == TREE_DESCR_8_7);
   1149    tl_assert(DESCR(0,0,0,0,0,0,0,1, 0,0,0, 0, 0,0,0) == TREE_DESCR_8_0);
   1150    tl_assert(DESCR(0,0,0,0,0,0,0,0, 1,0,0, 0, 0,0,0) == TREE_DESCR_16_3);
   1151    tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 0,0,0) == TREE_DESCR_32_1);
   1152    tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,1, 0, 0,0,0) == TREE_DESCR_16_2);
   1153    tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 1, 0,0,0) == TREE_DESCR_64);
   1154    tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 1,0,0) == TREE_DESCR_16_1);
   1155    tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 0,1,0) == TREE_DESCR_32_0);
   1156    tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 0,0,1) == TREE_DESCR_16_0);
   1157 
   1158    switch (descr) {
   1159    /*
   1160               +--------------------------------- TREE_DESCR_8_7
   1161               |             +------------------- TREE_DESCR_8_0
   1162               |             |  +---------------- TREE_DESCR_16_3
   1163               |             |  | +-------------- TREE_DESCR_32_1
   1164               |             |  | | +------------ TREE_DESCR_16_2
   1165               |             |  | | |  +--------- TREE_DESCR_64
   1166               |             |  | | |  |  +------ TREE_DESCR_16_1
   1167               |             |  | | |  |  | +---- TREE_DESCR_32_0
   1168               |             |  | | |  |  | | +-- TREE_DESCR_16_0
   1169               |             |  | | |  |  | | |
   1170               |             |  | | |  |  | | |   GRANULARITY, 7 -> 0 */
   1171    case DESCR(1,1,1,1,1,1,1,1, 0,0,0, 0, 0,0,0): /* 8 8 8 8  8 8 8 8 */
   1172                                                  return BYTE(1,1,1,1,1,1,1,1);
   1173    case DESCR(1,1,0,0,1,1,1,1, 0,0,1, 0, 0,0,0): /* 8 8 16   8 8 8 8 */
   1174                                                  return BYTE(1,1,0,1,1,1,1,1);
   1175    case DESCR(0,0,1,1,1,1,1,1, 1,0,0, 0, 0,0,0): /* 16  8 8  8 8 8 8 */
   1176                                                  return BYTE(0,1,1,1,1,1,1,1);
   1177    case DESCR(0,0,0,0,1,1,1,1, 1,0,1, 0, 0,0,0): /* 16  16   8 8 8 8 */
   1178                                                  return BYTE(0,1,0,1,1,1,1,1);
   1179 
   1180    case DESCR(1,1,1,1,1,1,0,0, 0,0,0, 0, 0,0,1): /* 8 8 8 8  8 8 16 */
   1181                                                  return BYTE(1,1,1,1,1,1,0,1);
   1182    case DESCR(1,1,0,0,1,1,0,0, 0,0,1, 0, 0,0,1): /* 8 8 16   8 8 16 */
   1183                                                  return BYTE(1,1,0,1,1,1,0,1);
   1184    case DESCR(0,0,1,1,1,1,0,0, 1,0,0, 0, 0,0,1): /* 16  8 8  8 8 16 */
   1185                                                  return BYTE(0,1,1,1,1,1,0,1);
   1186    case DESCR(0,0,0,0,1,1,0,0, 1,0,1, 0, 0,0,1): /* 16  16   8 8 16 */
   1187                                                  return BYTE(0,1,0,1,1,1,0,1);
   1188 
   1189    case DESCR(1,1,1,1,0,0,1,1, 0,0,0, 0, 1,0,0): /* 8 8 8 8  16 8 8 */
   1190                                                  return BYTE(1,1,1,1,0,1,1,1);
   1191    case DESCR(1,1,0,0,0,0,1,1, 0,0,1, 0, 1,0,0): /* 8 8 16   16 8 8 */
   1192                                                  return BYTE(1,1,0,1,0,1,1,1);
   1193    case DESCR(0,0,1,1,0,0,1,1, 1,0,0, 0, 1,0,0): /* 16  8 8  16 8 8 */
   1194                                                  return BYTE(0,1,1,1,0,1,1,1);
   1195    case DESCR(0,0,0,0,0,0,1,1, 1,0,1, 0, 1,0,0): /* 16  16   16 8 8 */
   1196                                                  return BYTE(0,1,0,1,0,1,1,1);
   1197 
   1198    case DESCR(1,1,1,1,0,0,0,0, 0,0,0, 0, 1,0,1): /* 8 8 8 8  16 16 */
   1199                                                  return BYTE(1,1,1,1,0,1,0,1);
   1200    case DESCR(1,1,0,0,0,0,0,0, 0,0,1, 0, 1,0,1): /* 8 8 16   16 16 */
   1201                                                  return BYTE(1,1,0,1,0,1,0,1);
   1202    case DESCR(0,0,1,1,0,0,0,0, 1,0,0, 0, 1,0,1): /* 16  8 8  16 16 */
   1203                                                  return BYTE(0,1,1,1,0,1,0,1);
   1204    case DESCR(0,0,0,0,0,0,0,0, 1,0,1, 0, 1,0,1): /* 16  16   16 16 */
   1205                                                  return BYTE(0,1,0,1,0,1,0,1);
   1206 
   1207    case DESCR(0,0,0,0,1,1,1,1, 0,1,0, 0, 0,0,0): /* 32  8 8 8 8 */
   1208                                                  return BYTE(0,0,0,1,1,1,1,1);
   1209    case DESCR(0,0,0,0,1,1,0,0, 0,1,0, 0, 0,0,1): /* 32  8 8 16  */
   1210                                                  return BYTE(0,0,0,1,1,1,0,1);
   1211    case DESCR(0,0,0,0,0,0,1,1, 0,1,0, 0, 1,0,0): /* 32  16  8 8 */
   1212                                                  return BYTE(0,0,0,1,0,1,1,1);
   1213    case DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 1,0,1): /* 32  16  16  */
   1214                                                  return BYTE(0,0,0,1,0,1,0,1);
   1215 
   1216    case DESCR(1,1,1,1,0,0,0,0, 0,0,0, 0, 0,1,0): /* 8 8 8 8  32 */
   1217                                                  return BYTE(1,1,1,1,0,0,0,1);
   1218    case DESCR(1,1,0,0,0,0,0,0, 0,0,1, 0, 0,1,0): /* 8 8 16   32 */
   1219                                                  return BYTE(1,1,0,1,0,0,0,1);
   1220    case DESCR(0,0,1,1,0,0,0,0, 1,0,0, 0, 0,1,0): /* 16  8 8  32 */
   1221                                                  return BYTE(0,1,1,1,0,0,0,1);
   1222    case DESCR(0,0,0,0,0,0,0,0, 1,0,1, 0, 0,1,0): /* 16  16   32 */
   1223                                                  return BYTE(0,1,0,1,0,0,0,1);
   1224 
   1225    case DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 0,1,0): /* 32 32 */
   1226                                                  return BYTE(0,0,0,1,0,0,0,1);
   1227 
   1228    case DESCR(0,0,0,0,0,0,0,0, 0,0,0, 1, 0,0,0): /* 64 */
   1229                                                  return BYTE(0,0,0,0,0,0,0,1);
   1230 
   1231    default: return BYTE(0,0,0,0,0,0,0,0);
   1232                    /* INVALID - any valid descr produces at least one
   1233                       valid bit in tree[0..7]*/
   1234    }
   1235    /* NOTREACHED*/
   1236    tl_assert(0);
   1237 
   1238 #  undef DESCR
   1239 #  undef BYTE
   1240 }
   1241 
   1242 __attribute__((unused))
   1243 static Bool is_sane_Descr ( UShort descr ) {
   1244    return descr_to_validbits(descr) != 0;
   1245 }
   1246 
   1247 static void sprintf_Descr ( /*OUT*/HChar* dst, UShort descr ) {
   1248    VG_(sprintf)(dst,
   1249                 "%d%d%d%d%d%d%d%d %d%d%d %d %d%d%d",
   1250                 (Int)((descr & TREE_DESCR_8_7) ? 1 : 0),
   1251                 (Int)((descr & TREE_DESCR_8_6) ? 1 : 0),
   1252                 (Int)((descr & TREE_DESCR_8_5) ? 1 : 0),
   1253                 (Int)((descr & TREE_DESCR_8_4) ? 1 : 0),
   1254                 (Int)((descr & TREE_DESCR_8_3) ? 1 : 0),
   1255                 (Int)((descr & TREE_DESCR_8_2) ? 1 : 0),
   1256                 (Int)((descr & TREE_DESCR_8_1) ? 1 : 0),
   1257                 (Int)((descr & TREE_DESCR_8_0) ? 1 : 0),
   1258                 (Int)((descr & TREE_DESCR_16_3) ? 1 : 0),
   1259                 (Int)((descr & TREE_DESCR_32_1) ? 1 : 0),
   1260                 (Int)((descr & TREE_DESCR_16_2) ? 1 : 0),
   1261                 (Int)((descr & TREE_DESCR_64)   ? 1 : 0),
   1262                 (Int)((descr & TREE_DESCR_16_1) ? 1 : 0),
   1263                 (Int)((descr & TREE_DESCR_32_0) ? 1 : 0),
   1264                 (Int)((descr & TREE_DESCR_16_0) ? 1 : 0)
   1265    );
   1266 }
   1267 static void sprintf_Byte ( /*OUT*/HChar* dst, UChar byte ) {
   1268    VG_(sprintf)(dst, "%d%d%d%d%d%d%d%d",
   1269                      (Int)((byte & 128) ? 1 : 0),
   1270                      (Int)((byte &  64) ? 1 : 0),
   1271                      (Int)((byte &  32) ? 1 : 0),
   1272                      (Int)((byte &  16) ? 1 : 0),
   1273                      (Int)((byte &   8) ? 1 : 0),
   1274                      (Int)((byte &   4) ? 1 : 0),
   1275                      (Int)((byte &   2) ? 1 : 0),
   1276                      (Int)((byte &   1) ? 1 : 0)
   1277    );
   1278 }
   1279 
   1280 static Bool is_sane_Descr_and_Tree ( UShort descr, SVal* tree ) {
   1281    Word  i;
   1282    UChar validbits = descr_to_validbits(descr);
   1283    HChar buf[128], buf2[128];    // large enough
   1284    if (validbits == 0)
   1285       goto bad;
   1286    for (i = 0; i < 8; i++) {
   1287       if (validbits & (1<<i)) {
   1288          if (tree[i] == SVal_INVALID)
   1289             goto bad;
   1290       } else {
   1291          if (tree[i] != SVal_INVALID)
   1292             goto bad;
   1293       }
   1294    }
   1295    return True;
   1296   bad:
   1297    sprintf_Descr( buf, descr );
   1298    sprintf_Byte( buf2, validbits );
   1299    VG_(printf)("%s","is_sane_Descr_and_Tree: bad tree {\n");
   1300    VG_(printf)("   validbits 0x%02lx    %s\n", (UWord)validbits, buf2);
   1301    VG_(printf)("       descr 0x%04lx  %s\n", (UWord)descr, buf);
   1302    for (i = 0; i < 8; i++)
   1303       VG_(printf)("   [%ld] 0x%016llx\n", i, tree[i]);
   1304    VG_(printf)("%s","}\n");
   1305    return 0;
   1306 }
   1307 
   1308 static Bool is_sane_CacheLine ( CacheLine* cl )
   1309 {
   1310    Word tno, cloff;
   1311 
   1312    if (!cl) goto bad;
   1313 
   1314    for (tno = 0, cloff = 0;  tno < N_LINE_TREES;  tno++, cloff += 8) {
   1315       UShort descr = cl->descrs[tno];
   1316       SVal*  tree  = &cl->svals[cloff];
   1317       if (!is_sane_Descr_and_Tree(descr, tree))
   1318          goto bad;
   1319    }
   1320    tl_assert(cloff == N_LINE_ARANGE);
   1321    return True;
   1322   bad:
   1323    pp_CacheLine(cl);
   1324    return False;
   1325 }
   1326 
   1327 static UShort normalise_tree ( /*MOD*/SVal* tree )
   1328 {
   1329    UShort descr;
   1330    /* pre: incoming tree[0..7] does not have any invalid shvals, in
   1331       particular no zeroes. */
   1332    if (CHECK_ZSM
   1333        && UNLIKELY(tree[7] == SVal_INVALID || tree[6] == SVal_INVALID
   1334                    || tree[5] == SVal_INVALID || tree[4] == SVal_INVALID
   1335                    || tree[3] == SVal_INVALID || tree[2] == SVal_INVALID
   1336                    || tree[1] == SVal_INVALID || tree[0] == SVal_INVALID))
   1337       tl_assert(0);
   1338 
   1339    descr = TREE_DESCR_8_7 | TREE_DESCR_8_6 | TREE_DESCR_8_5
   1340            | TREE_DESCR_8_4 | TREE_DESCR_8_3 | TREE_DESCR_8_2
   1341            | TREE_DESCR_8_1 | TREE_DESCR_8_0;
   1342    /* build 16-bit layer */
   1343    if (tree[1] == tree[0]) {
   1344       tree[1] = SVal_INVALID;
   1345       descr &= ~(TREE_DESCR_8_1 | TREE_DESCR_8_0);
   1346       descr |= TREE_DESCR_16_0;
   1347    }
   1348    if (tree[3] == tree[2]) {
   1349       tree[3] = SVal_INVALID;
   1350       descr &= ~(TREE_DESCR_8_3 | TREE_DESCR_8_2);
   1351       descr |= TREE_DESCR_16_1;
   1352    }
   1353    if (tree[5] == tree[4]) {
   1354       tree[5] = SVal_INVALID;
   1355       descr &= ~(TREE_DESCR_8_5 | TREE_DESCR_8_4);
   1356       descr |= TREE_DESCR_16_2;
   1357    }
   1358    if (tree[7] == tree[6]) {
   1359       tree[7] = SVal_INVALID;
   1360       descr &= ~(TREE_DESCR_8_7 | TREE_DESCR_8_6);
   1361       descr |= TREE_DESCR_16_3;
   1362    }
   1363    /* build 32-bit layer */
   1364    if (tree[2] == tree[0]
   1365        && (descr & TREE_DESCR_16_1) && (descr & TREE_DESCR_16_0)) {
   1366       tree[2] = SVal_INVALID; /* [3,1] must already be SVal_INVALID */
   1367       descr &= ~(TREE_DESCR_16_1 | TREE_DESCR_16_0);
   1368       descr |= TREE_DESCR_32_0;
   1369    }
   1370    if (tree[6] == tree[4]
   1371        && (descr & TREE_DESCR_16_3) && (descr & TREE_DESCR_16_2)) {
   1372       tree[6] = SVal_INVALID; /* [7,5] must already be SVal_INVALID */
   1373       descr &= ~(TREE_DESCR_16_3 | TREE_DESCR_16_2);
   1374       descr |= TREE_DESCR_32_1;
   1375    }
   1376    /* build 64-bit layer */
   1377    if (tree[4] == tree[0]
   1378        && (descr & TREE_DESCR_32_1) && (descr & TREE_DESCR_32_0)) {
   1379       tree[4] = SVal_INVALID; /* [7,6,5,3,2,1] must already be SVal_INVALID */
   1380       descr &= ~(TREE_DESCR_32_1 | TREE_DESCR_32_0);
   1381       descr |= TREE_DESCR_64;
   1382    }
   1383    return descr;
   1384 }
   1385 
   1386 /* This takes a cacheline where all the data is at the leaves
   1387    (w8[..]) and builds a correctly normalised tree. */
   1388 static void normalise_CacheLine ( /*MOD*/CacheLine* cl )
   1389 {
   1390    Word tno, cloff;
   1391    for (tno = 0, cloff = 0;  tno < N_LINE_TREES;  tno++, cloff += 8) {
   1392       SVal* tree = &cl->svals[cloff];
   1393       cl->descrs[tno] = normalise_tree( tree );
   1394    }
   1395    tl_assert(cloff == N_LINE_ARANGE);
   1396    if (CHECK_ZSM)
   1397       tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   1398    stats__cline_normalises++;
   1399 }
   1400 
   1401 
   1402 typedef struct { UChar count; SVal sval; } CountedSVal;
   1403 
   1404 static
   1405 void sequentialise_CacheLine ( /*OUT*/CountedSVal* dst,
   1406                                /*OUT*/Word* dstUsedP,
   1407                                Word nDst, CacheLine* src )
   1408 {
   1409    Word  tno, cloff, dstUsed;
   1410 
   1411    tl_assert(nDst == N_LINE_ARANGE);
   1412    dstUsed = 0;
   1413 
   1414    for (tno = 0, cloff = 0;  tno < N_LINE_TREES;  tno++, cloff += 8) {
   1415       UShort descr = src->descrs[tno];
   1416       SVal*  tree  = &src->svals[cloff];
   1417 
   1418       /* sequentialise the tree described by (descr,tree). */
   1419 #     define PUT(_n,_v)                                \
   1420          do { dst[dstUsed  ].count = (_n);             \
   1421               dst[dstUsed++].sval  = (_v);             \
   1422          } while (0)
   1423 
   1424       /* byte 0 */
   1425       if (descr & TREE_DESCR_64)   PUT(8, tree[0]); else
   1426       if (descr & TREE_DESCR_32_0) PUT(4, tree[0]); else
   1427       if (descr & TREE_DESCR_16_0) PUT(2, tree[0]); else
   1428       if (descr & TREE_DESCR_8_0)  PUT(1, tree[0]);
   1429       /* byte 1 */
   1430       if (descr & TREE_DESCR_8_1)  PUT(1, tree[1]);
   1431       /* byte 2 */
   1432       if (descr & TREE_DESCR_16_1) PUT(2, tree[2]); else
   1433       if (descr & TREE_DESCR_8_2)  PUT(1, tree[2]);
   1434       /* byte 3 */
   1435       if (descr & TREE_DESCR_8_3)  PUT(1, tree[3]);
   1436       /* byte 4 */
   1437       if (descr & TREE_DESCR_32_1) PUT(4, tree[4]); else
   1438       if (descr & TREE_DESCR_16_2) PUT(2, tree[4]); else
   1439       if (descr & TREE_DESCR_8_4)  PUT(1, tree[4]);
   1440       /* byte 5 */
   1441       if (descr & TREE_DESCR_8_5)  PUT(1, tree[5]);
   1442       /* byte 6 */
   1443       if (descr & TREE_DESCR_16_3) PUT(2, tree[6]); else
   1444       if (descr & TREE_DESCR_8_6)  PUT(1, tree[6]);
   1445       /* byte 7 */
   1446       if (descr & TREE_DESCR_8_7)  PUT(1, tree[7]);
   1447 
   1448 #     undef PUT
   1449       /* END sequentialise the tree described by (descr,tree). */
   1450 
   1451    }
   1452    tl_assert(cloff == N_LINE_ARANGE);
   1453    tl_assert(dstUsed <= nDst);
   1454 
   1455    *dstUsedP = dstUsed;
   1456 }
   1457 
   1458 /* Write the cacheline 'wix' to backing store.  Where it ends up
   1459    is determined by its tag field. */
   1460 static __attribute__((noinline)) void cacheline_wback ( UWord wix )
   1461 {
   1462    Word        i, j, k, m;
   1463    Addr        tag;
   1464    SecMap*     sm;
   1465    CacheLine*  cl;
   1466    LineZ* lineZ;
   1467    LineF* lineF;
   1468    Word        zix, fix, csvalsUsed;
   1469    CountedSVal csvals[N_LINE_ARANGE];
   1470    SVal        sv;
   1471 
   1472    if (0)
   1473    VG_(printf)("scache wback line %d\n", (Int)wix);
   1474 
   1475    tl_assert(wix >= 0 && wix < N_WAY_NENT);
   1476 
   1477    tag =  cache_shmem.tags0[wix];
   1478    cl  = &cache_shmem.lyns0[wix];
   1479 
   1480    /* The cache line may have been invalidated; if so, ignore it. */
   1481    if (!is_valid_scache_tag(tag))
   1482       return;
   1483 
   1484    /* Where are we going to put it? */
   1485    sm         = NULL;
   1486    lineZ      = NULL;
   1487    lineF      = NULL;
   1488    zix = fix = -1;
   1489 
   1490    /* find the Z line to write in and rcdec it or the associated F
   1491       line. */
   1492    find_Z_for_writing( &sm, &zix, tag );
   1493 
   1494    tl_assert(sm);
   1495    tl_assert(zix >= 0 && zix < N_SECMAP_ZLINES);
   1496    lineZ = &sm->linesZ[zix];
   1497 
   1498    /* Generate the data to be stored */
   1499    if (CHECK_ZSM)
   1500       tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   1501 
   1502    csvalsUsed = -1;
   1503    sequentialise_CacheLine( csvals, &csvalsUsed,
   1504                             N_LINE_ARANGE, cl );
   1505    tl_assert(csvalsUsed >= 1 && csvalsUsed <= N_LINE_ARANGE);
   1506    if (0) VG_(printf)("%ld ", csvalsUsed);
   1507 
   1508    lineZ->dict[0] = lineZ->dict[1]
   1509                   = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID;
   1510 
   1511    /* i indexes actual shadow values, k is cursor in csvals */
   1512    i = 0;
   1513    for (k = 0; k < csvalsUsed; k++) {
   1514 
   1515       sv = csvals[k].sval;
   1516       if (CHECK_ZSM)
   1517          tl_assert(csvals[k].count >= 1 && csvals[k].count <= 8);
   1518       /* do we already have it? */
   1519       if (sv == lineZ->dict[0]) { j = 0; goto dict_ok; }
   1520       if (sv == lineZ->dict[1]) { j = 1; goto dict_ok; }
   1521       if (sv == lineZ->dict[2]) { j = 2; goto dict_ok; }
   1522       if (sv == lineZ->dict[3]) { j = 3; goto dict_ok; }
   1523       /* no.  look for a free slot. */
   1524       if (CHECK_ZSM)
   1525          tl_assert(sv != SVal_INVALID);
   1526       if (lineZ->dict[0]
   1527           == SVal_INVALID) { lineZ->dict[0] = sv; j = 0; goto dict_ok; }
   1528       if (lineZ->dict[1]
   1529           == SVal_INVALID) { lineZ->dict[1] = sv; j = 1; goto dict_ok; }
   1530       if (lineZ->dict[2]
   1531           == SVal_INVALID) { lineZ->dict[2] = sv; j = 2; goto dict_ok; }
   1532       if (lineZ->dict[3]
   1533           == SVal_INVALID) { lineZ->dict[3] = sv; j = 3; goto dict_ok; }
   1534       break; /* we'll have to use the f rep */
   1535      dict_ok:
   1536       m = csvals[k].count;
   1537       if (m == 8) {
   1538          write_twobit_array( lineZ->ix2s, i+0, j );
   1539          write_twobit_array( lineZ->ix2s, i+1, j );
   1540          write_twobit_array( lineZ->ix2s, i+2, j );
   1541          write_twobit_array( lineZ->ix2s, i+3, j );
   1542          write_twobit_array( lineZ->ix2s, i+4, j );
   1543          write_twobit_array( lineZ->ix2s, i+5, j );
   1544          write_twobit_array( lineZ->ix2s, i+6, j );
   1545          write_twobit_array( lineZ->ix2s, i+7, j );
   1546          i += 8;
   1547       }
   1548       else if (m == 4) {
   1549          write_twobit_array( lineZ->ix2s, i+0, j );
   1550          write_twobit_array( lineZ->ix2s, i+1, j );
   1551          write_twobit_array( lineZ->ix2s, i+2, j );
   1552          write_twobit_array( lineZ->ix2s, i+3, j );
   1553          i += 4;
   1554       }
   1555       else if (m == 1) {
   1556          write_twobit_array( lineZ->ix2s, i+0, j );
   1557          i += 1;
   1558       }
   1559       else if (m == 2) {
   1560          write_twobit_array( lineZ->ix2s, i+0, j );
   1561          write_twobit_array( lineZ->ix2s, i+1, j );
   1562          i += 2;
   1563       }
   1564       else {
   1565          tl_assert(0); /* 8 4 2 or 1 are the only legitimate values for m */
   1566       }
   1567 
   1568    }
   1569 
   1570    if (LIKELY(i == N_LINE_ARANGE)) {
   1571       /* Construction of the compressed representation was
   1572          successful. */
   1573       rcinc_LineZ(lineZ);
   1574       stats__cache_Z_wbacks++;
   1575    } else {
   1576       /* Cannot use the compressed(z) representation.  Use the full(f)
   1577          rep instead. */
   1578       tl_assert(i >= 0 && i < N_LINE_ARANGE);
   1579       lineZ->dict[0] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID;
   1580       lineF = alloc_LineF_for_Z (lineZ);
   1581       i = 0;
   1582       for (k = 0; k < csvalsUsed; k++) {
   1583          if (CHECK_ZSM)
   1584             tl_assert(csvals[k].count >= 1 && csvals[k].count <= 8);
   1585          sv = csvals[k].sval;
   1586          if (CHECK_ZSM)
   1587             tl_assert(sv != SVal_INVALID);
   1588          for (m = csvals[k].count; m > 0; m--) {
   1589             lineF->w64s[i] = sv;
   1590             i++;
   1591          }
   1592       }
   1593       tl_assert(i == N_LINE_ARANGE);
   1594       rcinc_LineF(lineF);
   1595       stats__cache_F_wbacks++;
   1596    }
   1597 }
   1598 
   1599 /* Fetch the cacheline 'wix' from the backing store.  The tag
   1600    associated with 'wix' is assumed to have already been filled in;
   1601    hence that is used to determine where in the backing store to read
   1602    from. */
   1603 static __attribute__((noinline)) void cacheline_fetch ( UWord wix )
   1604 {
   1605    Word       i;
   1606    Addr       tag;
   1607    CacheLine* cl;
   1608    LineZ*     lineZ;
   1609    LineF*     lineF;
   1610 
   1611    if (0)
   1612    VG_(printf)("scache fetch line %d\n", (Int)wix);
   1613 
   1614    tl_assert(wix >= 0 && wix < N_WAY_NENT);
   1615 
   1616    tag =  cache_shmem.tags0[wix];
   1617    cl  = &cache_shmem.lyns0[wix];
   1618 
   1619    /* reject nonsense requests */
   1620    tl_assert(is_valid_scache_tag(tag));
   1621 
   1622    lineZ = NULL;
   1623    lineF = NULL;
   1624    find_ZF_for_reading( &lineZ, &lineF, tag );
   1625    tl_assert( (lineZ && !lineF) || (!lineZ && lineF) );
   1626 
   1627    /* expand the data into the bottom layer of the tree, then get
   1628       cacheline_normalise to build the descriptor array. */
   1629    if (lineF) {
   1630       for (i = 0; i < N_LINE_ARANGE; i++) {
   1631          cl->svals[i] = lineF->w64s[i];
   1632       }
   1633       stats__cache_F_fetches++;
   1634    } else {
   1635       for (i = 0; i < N_LINE_ARANGE; i++) {
   1636          UWord ix = read_twobit_array( lineZ->ix2s, i );
   1637          if (CHECK_ZSM) tl_assert(ix >= 0 && ix <= 3);
   1638          cl->svals[i] = lineZ->dict[ix];
   1639          if (CHECK_ZSM) tl_assert(cl->svals[i] != SVal_INVALID);
   1640       }
   1641       stats__cache_Z_fetches++;
   1642    }
   1643    normalise_CacheLine( cl );
   1644 }
   1645 
   1646 /* Invalid the cachelines corresponding to the given range, which
   1647    must start and end on a cacheline boundary. */
   1648 static void shmem__invalidate_scache_range (Addr ga, SizeT szB)
   1649 {
   1650    Word wix;
   1651 
   1652    /* ga must be on a cacheline boundary. */
   1653    tl_assert (is_valid_scache_tag (ga));
   1654    /* szB must be a multiple of cacheline size. */
   1655    tl_assert (0 == (szB & (N_LINE_ARANGE - 1)));
   1656 
   1657 
   1658    Word ga_ix = (ga >> N_LINE_BITS) & (N_WAY_NENT - 1);
   1659    Word nwix = szB / N_LINE_ARANGE;
   1660 
   1661    if (nwix > N_WAY_NENT)
   1662       nwix = N_WAY_NENT; // no need to check several times the same entry.
   1663 
   1664    for (wix = 0; wix < nwix; wix++) {
   1665       if (address_in_range(cache_shmem.tags0[ga_ix], ga, szB))
   1666          cache_shmem.tags0[ga_ix] = 1/*INVALID*/;
   1667       ga_ix++;
   1668       if (UNLIKELY(ga_ix == N_WAY_NENT))
   1669          ga_ix = 0;
   1670    }
   1671 }
   1672 
   1673 
   1674 static void shmem__flush_and_invalidate_scache ( void ) {
   1675    Word wix;
   1676    Addr tag;
   1677    if (0) VG_(printf)("%s","scache flush and invalidate\n");
   1678    tl_assert(!is_valid_scache_tag(1));
   1679    for (wix = 0; wix < N_WAY_NENT; wix++) {
   1680       tag = cache_shmem.tags0[wix];
   1681       if (tag == 1/*INVALID*/) {
   1682          /* already invalid; nothing to do */
   1683       } else {
   1684          tl_assert(is_valid_scache_tag(tag));
   1685          cacheline_wback( wix );
   1686       }
   1687       cache_shmem.tags0[wix] = 1/*INVALID*/;
   1688    }
   1689    stats__cache_flushes_invals++;
   1690 }
   1691 
   1692 
   1693 static inline Bool aligned16 ( Addr a ) {
   1694    return 0 == (a & 1);
   1695 }
   1696 static inline Bool aligned32 ( Addr a ) {
   1697    return 0 == (a & 3);
   1698 }
   1699 static inline Bool aligned64 ( Addr a ) {
   1700    return 0 == (a & 7);
   1701 }
   1702 static inline UWord get_cacheline_offset ( Addr a ) {
   1703    return (UWord)(a & (N_LINE_ARANGE - 1));
   1704 }
   1705 static inline Addr cacheline_ROUNDUP ( Addr a ) {
   1706    return ROUNDUP(a, N_LINE_ARANGE);
   1707 }
   1708 static inline Addr cacheline_ROUNDDN ( Addr a ) {
   1709    return ROUNDDN(a, N_LINE_ARANGE);
   1710 }
   1711 static inline UWord get_treeno ( Addr a ) {
   1712    return get_cacheline_offset(a) >> 3;
   1713 }
   1714 static inline UWord get_tree_offset ( Addr a ) {
   1715    return a & 7;
   1716 }
   1717 
   1718 static __attribute__((noinline))
   1719        CacheLine* get_cacheline_MISS ( Addr a ); /* fwds */
   1720 static inline CacheLine* get_cacheline ( Addr a )
   1721 {
   1722    /* tag is 'a' with the in-line offset masked out,
   1723       eg a[31]..a[4] 0000 */
   1724    Addr       tag = a & ~(N_LINE_ARANGE - 1);
   1725    UWord      wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1);
   1726    stats__cache_totrefs++;
   1727    if (LIKELY(tag == cache_shmem.tags0[wix])) {
   1728       return &cache_shmem.lyns0[wix];
   1729    } else {
   1730       return get_cacheline_MISS( a );
   1731    }
   1732 }
   1733 
   1734 static __attribute__((noinline))
   1735        CacheLine* get_cacheline_MISS ( Addr a )
   1736 {
   1737    /* tag is 'a' with the in-line offset masked out,
   1738       eg a[31]..a[4] 0000 */
   1739 
   1740    CacheLine* cl;
   1741    Addr*      tag_old_p;
   1742    Addr       tag = a & ~(N_LINE_ARANGE - 1);
   1743    UWord      wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1);
   1744 
   1745    tl_assert(tag != cache_shmem.tags0[wix]);
   1746 
   1747    /* Dump the old line into the backing store. */
   1748    stats__cache_totmisses++;
   1749 
   1750    cl        = &cache_shmem.lyns0[wix];
   1751    tag_old_p = &cache_shmem.tags0[wix];
   1752 
   1753    if (is_valid_scache_tag( *tag_old_p )) {
   1754       /* EXPENSIVE and REDUNDANT: callee does it */
   1755       if (CHECK_ZSM)
   1756          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   1757       cacheline_wback( wix );
   1758    }
   1759    /* and reload the new one */
   1760    *tag_old_p = tag;
   1761    cacheline_fetch( wix );
   1762    if (CHECK_ZSM)
   1763       tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   1764    return cl;
   1765 }
   1766 
   1767 static UShort pulldown_to_32 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) {
   1768    stats__cline_64to32pulldown++;
   1769    switch (toff) {
   1770       case 0: case 4:
   1771          tl_assert(descr & TREE_DESCR_64);
   1772          tree[4] = tree[0];
   1773          descr &= ~TREE_DESCR_64;
   1774          descr |= (TREE_DESCR_32_1 | TREE_DESCR_32_0);
   1775          break;
   1776       default:
   1777          tl_assert(0);
   1778    }
   1779    return descr;
   1780 }
   1781 
   1782 static UShort pulldown_to_16 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) {
   1783    stats__cline_32to16pulldown++;
   1784    switch (toff) {
   1785       case 0: case 2:
   1786          if (!(descr & TREE_DESCR_32_0)) {
   1787             descr = pulldown_to_32(tree, 0, descr);
   1788          }
   1789          tl_assert(descr & TREE_DESCR_32_0);
   1790          tree[2] = tree[0];
   1791          descr &= ~TREE_DESCR_32_0;
   1792          descr |= (TREE_DESCR_16_1 | TREE_DESCR_16_0);
   1793          break;
   1794       case 4: case 6:
   1795          if (!(descr & TREE_DESCR_32_1)) {
   1796             descr = pulldown_to_32(tree, 4, descr);
   1797          }
   1798          tl_assert(descr & TREE_DESCR_32_1);
   1799          tree[6] = tree[4];
   1800          descr &= ~TREE_DESCR_32_1;
   1801          descr |= (TREE_DESCR_16_3 | TREE_DESCR_16_2);
   1802          break;
   1803       default:
   1804          tl_assert(0);
   1805    }
   1806    return descr;
   1807 }
   1808 
   1809 static UShort pulldown_to_8 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) {
   1810    stats__cline_16to8pulldown++;
   1811    switch (toff) {
   1812       case 0: case 1:
   1813          if (!(descr & TREE_DESCR_16_0)) {
   1814             descr = pulldown_to_16(tree, 0, descr);
   1815          }
   1816          tl_assert(descr & TREE_DESCR_16_0);
   1817          tree[1] = tree[0];
   1818          descr &= ~TREE_DESCR_16_0;
   1819          descr |= (TREE_DESCR_8_1 | TREE_DESCR_8_0);
   1820          break;
   1821       case 2: case 3:
   1822          if (!(descr & TREE_DESCR_16_1)) {
   1823             descr = pulldown_to_16(tree, 2, descr);
   1824          }
   1825          tl_assert(descr & TREE_DESCR_16_1);
   1826          tree[3] = tree[2];
   1827          descr &= ~TREE_DESCR_16_1;
   1828          descr |= (TREE_DESCR_8_3 | TREE_DESCR_8_2);
   1829          break;
   1830       case 4: case 5:
   1831          if (!(descr & TREE_DESCR_16_2)) {
   1832             descr = pulldown_to_16(tree, 4, descr);
   1833          }
   1834          tl_assert(descr & TREE_DESCR_16_2);
   1835          tree[5] = tree[4];
   1836          descr &= ~TREE_DESCR_16_2;
   1837          descr |= (TREE_DESCR_8_5 | TREE_DESCR_8_4);
   1838          break;
   1839       case 6: case 7:
   1840          if (!(descr & TREE_DESCR_16_3)) {
   1841             descr = pulldown_to_16(tree, 6, descr);
   1842          }
   1843          tl_assert(descr & TREE_DESCR_16_3);
   1844          tree[7] = tree[6];
   1845          descr &= ~TREE_DESCR_16_3;
   1846          descr |= (TREE_DESCR_8_7 | TREE_DESCR_8_6);
   1847          break;
   1848       default:
   1849          tl_assert(0);
   1850    }
   1851    return descr;
   1852 }
   1853 
   1854 
   1855 static UShort pullup_descr_to_16 ( UShort descr, UWord toff ) {
   1856    UShort mask;
   1857    switch (toff) {
   1858       case 0:
   1859          mask = TREE_DESCR_8_1 | TREE_DESCR_8_0;
   1860          tl_assert( (descr & mask) == mask );
   1861          descr &= ~mask;
   1862          descr |= TREE_DESCR_16_0;
   1863          break;
   1864       case 2:
   1865          mask = TREE_DESCR_8_3 | TREE_DESCR_8_2;
   1866          tl_assert( (descr & mask) == mask );
   1867          descr &= ~mask;
   1868          descr |= TREE_DESCR_16_1;
   1869          break;
   1870       case 4:
   1871          mask = TREE_DESCR_8_5 | TREE_DESCR_8_4;
   1872          tl_assert( (descr & mask) == mask );
   1873          descr &= ~mask;
   1874          descr |= TREE_DESCR_16_2;
   1875          break;
   1876       case 6:
   1877          mask = TREE_DESCR_8_7 | TREE_DESCR_8_6;
   1878          tl_assert( (descr & mask) == mask );
   1879          descr &= ~mask;
   1880          descr |= TREE_DESCR_16_3;
   1881          break;
   1882       default:
   1883          tl_assert(0);
   1884    }
   1885    return descr;
   1886 }
   1887 
   1888 static UShort pullup_descr_to_32 ( UShort descr, UWord toff ) {
   1889    UShort mask;
   1890    switch (toff) {
   1891       case 0:
   1892          if (!(descr & TREE_DESCR_16_0))
   1893             descr = pullup_descr_to_16(descr, 0);
   1894          if (!(descr & TREE_DESCR_16_1))
   1895             descr = pullup_descr_to_16(descr, 2);
   1896          mask = TREE_DESCR_16_1 | TREE_DESCR_16_0;
   1897          tl_assert( (descr & mask) == mask );
   1898          descr &= ~mask;
   1899          descr |= TREE_DESCR_32_0;
   1900          break;
   1901       case 4:
   1902          if (!(descr & TREE_DESCR_16_2))
   1903             descr = pullup_descr_to_16(descr, 4);
   1904          if (!(descr & TREE_DESCR_16_3))
   1905             descr = pullup_descr_to_16(descr, 6);
   1906          mask = TREE_DESCR_16_3 | TREE_DESCR_16_2;
   1907          tl_assert( (descr & mask) == mask );
   1908          descr &= ~mask;
   1909          descr |= TREE_DESCR_32_1;
   1910          break;
   1911       default:
   1912          tl_assert(0);
   1913    }
   1914    return descr;
   1915 }
   1916 
   1917 static Bool valid_value_is_above_me_32 ( UShort descr, UWord toff ) {
   1918    switch (toff) {
   1919       case 0: case 4:
   1920          return 0 != (descr & TREE_DESCR_64);
   1921       default:
   1922          tl_assert(0);
   1923    }
   1924 }
   1925 
   1926 static Bool valid_value_is_below_me_16 ( UShort descr, UWord toff ) {
   1927    switch (toff) {
   1928       case 0:
   1929          return 0 != (descr & (TREE_DESCR_8_1 | TREE_DESCR_8_0));
   1930       case 2:
   1931          return 0 != (descr & (TREE_DESCR_8_3 | TREE_DESCR_8_2));
   1932       case 4:
   1933          return 0 != (descr & (TREE_DESCR_8_5 | TREE_DESCR_8_4));
   1934       case 6:
   1935          return 0 != (descr & (TREE_DESCR_8_7 | TREE_DESCR_8_6));
   1936       default:
   1937          tl_assert(0);
   1938    }
   1939 }
   1940 
   1941 /* ------------ Cache management ------------ */
   1942 
   1943 static void zsm_flush_cache ( void )
   1944 {
   1945    shmem__flush_and_invalidate_scache();
   1946 }
   1947 
   1948 
   1949 static void zsm_init ( void )
   1950 {
   1951    tl_assert( sizeof(UWord) == sizeof(Addr) );
   1952 
   1953    tl_assert(map_shmem == NULL);
   1954    map_shmem = VG_(newFM)( HG_(zalloc), "libhb.zsm_init.1 (map_shmem)",
   1955                            HG_(free),
   1956                            NULL/*unboxed UWord cmp*/);
   1957    /* Invalidate all cache entries. */
   1958    tl_assert(!is_valid_scache_tag(1));
   1959    for (UWord wix = 0; wix < N_WAY_NENT; wix++) {
   1960       cache_shmem.tags0[wix] = 1/*INVALID*/;
   1961    }
   1962 
   1963    LineF_pool_allocator = VG_(newPA) (
   1964                              sizeof(LineF),
   1965                              /* Nr elements/pool to fill a core arena block
   1966                                 taking some arena overhead into account. */
   1967                              (4 * 1024 * 1024 - 200)/sizeof(LineF),
   1968                              HG_(zalloc),
   1969                              "libhb.LineF_storage.pool",
   1970                              HG_(free)
   1971                           );
   1972 
   1973    /* a SecMap must contain an integral number of CacheLines */
   1974    tl_assert(0 == (N_SECMAP_ARANGE % N_LINE_ARANGE));
   1975    /* also ... a CacheLine holds an integral number of trees */
   1976    tl_assert(0 == (N_LINE_ARANGE % 8));
   1977 }
   1978 
   1979 /////////////////////////////////////////////////////////////////
   1980 /////////////////////////////////////////////////////////////////
   1981 //                                                             //
   1982 // SECTION END compressed shadow memory                        //
   1983 //                                                             //
   1984 /////////////////////////////////////////////////////////////////
   1985 /////////////////////////////////////////////////////////////////
   1986 
   1987 
   1988 
   1989 /////////////////////////////////////////////////////////////////
   1990 /////////////////////////////////////////////////////////////////
   1991 //                                                             //
   1992 // SECTION BEGIN vts primitives                                //
   1993 //                                                             //
   1994 /////////////////////////////////////////////////////////////////
   1995 /////////////////////////////////////////////////////////////////
   1996 
   1997 
   1998 /* There's a 1-1 mapping between Thr and ThrIDs -- the latter merely
   1999    being compact stand-ins for Thr*'s.  Use these functions to map
   2000    between them. */
   2001 static ThrID Thr__to_ThrID   ( Thr*  thr   ); /* fwds */
   2002 static Thr*  Thr__from_ThrID ( ThrID thrid ); /* fwds */
   2003 
   2004 __attribute__((noreturn))
   2005 static void scalarts_limitations_fail_NORETURN ( Bool due_to_nThrs )
   2006 {
   2007    if (due_to_nThrs) {
   2008       const HChar* s =
   2009          "\n"
   2010          "Helgrind: cannot continue, run aborted: too many threads.\n"
   2011          "Sorry.  Helgrind can only handle programs that create\n"
   2012          "%'llu or fewer threads over their entire lifetime.\n"
   2013          "\n";
   2014       VG_(umsg)(s, (ULong)(ThrID_MAX_VALID - 1024));
   2015    } else {
   2016       const HChar* s =
   2017          "\n"
   2018          "Helgrind: cannot continue, run aborted: too many\n"
   2019          "synchronisation events.  Sorry. Helgrind can only handle\n"
   2020          "programs which perform %'llu or fewer\n"
   2021          "inter-thread synchronisation events (locks, unlocks, etc).\n"
   2022          "\n";
   2023       VG_(umsg)(s, (1ULL << SCALARTS_N_TYMBITS) - 1);
   2024    }
   2025    VG_(exit)(1);
   2026    /*NOTREACHED*/
   2027    tl_assert(0); /*wtf?!*/
   2028 }
   2029 
   2030 
   2031 /* The dead thread (ThrID, actually) tables.  A thread may only be
   2032    listed here if we have been notified thereof by libhb_async_exit.
   2033    New entries are added at the end.  The order isn't important, but
   2034    the ThrID values must be unique.
   2035    verydead_thread_table_not_pruned lists the identity of the threads
   2036    that died since the previous round of pruning.
   2037    Once pruning is done, these ThrID are added in verydead_thread_table.
   2038    We don't actually need to keep the set of threads that have ever died --
   2039    only the threads that have died since the previous round of
   2040    pruning.  But it's useful for sanity check purposes to keep the
   2041    entire set, so we do. */
   2042 static XArray* /* of ThrID */ verydead_thread_table_not_pruned = NULL;
   2043 static XArray* /* of ThrID */ verydead_thread_table = NULL;
   2044 
   2045 /* Arbitrary total ordering on ThrIDs. */
   2046 static Int cmp__ThrID ( const void* v1, const void* v2 ) {
   2047    ThrID id1 = *(const ThrID*)v1;
   2048    ThrID id2 = *(const ThrID*)v2;
   2049    if (id1 < id2) return -1;
   2050    if (id1 > id2) return 1;
   2051    return 0;
   2052 }
   2053 
   2054 static void verydead_thread_tables_init ( void )
   2055 {
   2056    tl_assert(!verydead_thread_table);
   2057    tl_assert(!verydead_thread_table_not_pruned);
   2058    verydead_thread_table
   2059      = VG_(newXA)( HG_(zalloc),
   2060                    "libhb.verydead_thread_table_init.1",
   2061                    HG_(free), sizeof(ThrID) );
   2062    VG_(setCmpFnXA)(verydead_thread_table, cmp__ThrID);
   2063    verydead_thread_table_not_pruned
   2064      = VG_(newXA)( HG_(zalloc),
   2065                    "libhb.verydead_thread_table_init.2",
   2066                    HG_(free), sizeof(ThrID) );
   2067    VG_(setCmpFnXA)(verydead_thread_table_not_pruned, cmp__ThrID);
   2068 }
   2069 
   2070 static void verydead_thread_table_sort_and_check (XArray* thrids)
   2071 {
   2072    UWord i;
   2073 
   2074    VG_(sortXA)( thrids );
   2075    /* Sanity check: check for unique .sts.thr values. */
   2076    UWord nBT = VG_(sizeXA)( thrids );
   2077    if (nBT > 0) {
   2078       ThrID thrid1, thrid2;
   2079       thrid2 = *(ThrID*)VG_(indexXA)( thrids, 0 );
   2080       for (i = 1; i < nBT; i++) {
   2081          thrid1 = thrid2;
   2082          thrid2 = *(ThrID*)VG_(indexXA)( thrids, i );
   2083          tl_assert(thrid1 < thrid2);
   2084       }
   2085    }
   2086    /* Ok, so the dead thread table thrids has unique and in-order keys. */
   2087 }
   2088 
   2089 /* A VTS contains .ts, its vector clock, and also .id, a field to hold
   2090    a backlink for the caller's convenience.  Since we have no idea
   2091    what to set that to in the library, it always gets set to
   2092    VtsID_INVALID. */
   2093 typedef
   2094    struct {
   2095       VtsID    id;
   2096       UInt     usedTS;
   2097       UInt     sizeTS;
   2098       ScalarTS ts[0];
   2099    }
   2100    VTS;
   2101 
   2102 /* Allocate a VTS capable of storing 'sizeTS' entries. */
   2103 static VTS* VTS__new ( const HChar* who, UInt sizeTS );
   2104 
   2105 /* Make a clone of 'vts', sizing the new array to exactly match the
   2106    number of ScalarTSs present. */
   2107 static VTS* VTS__clone ( const HChar* who, VTS* vts );
   2108 
   2109 /* Make a clone of 'vts' with the thrids in 'thrids' removed.  The new
   2110    array is sized exactly to hold the number of required elements.
   2111    'thridsToDel' is an array of ThrIDs to be omitted in the clone, and
   2112    must be in strictly increasing order. */
   2113 static VTS* VTS__subtract ( const HChar* who, VTS* vts, XArray* thridsToDel );
   2114 
   2115 /* Delete this VTS in its entirety. */
   2116 static void VTS__delete ( VTS* vts );
   2117 
   2118 /* Create a new singleton VTS in 'out'.  Caller must have
   2119    pre-allocated 'out' sufficiently big to hold the result in all
   2120    possible cases. */
   2121 static void VTS__singleton ( /*OUT*/VTS* out, Thr* thr, ULong tym );
   2122 
   2123 /* Create in 'out' a VTS which is the same as 'vts' except with
   2124    vts[me]++, so to speak.  Caller must have pre-allocated 'out'
   2125    sufficiently big to hold the result in all possible cases. */
   2126 static void VTS__tick ( /*OUT*/VTS* out, Thr* me, VTS* vts );
   2127 
   2128 /* Create in 'out' a VTS which is the join (max) of 'a' and
   2129    'b'. Caller must have pre-allocated 'out' sufficiently big to hold
   2130    the result in all possible cases. */
   2131 static void VTS__join ( /*OUT*/VTS* out, VTS* a, VTS* b );
   2132 
   2133 /* Compute the partial ordering relation of the two args.  Although we
   2134    could be completely general and return an enumeration value (EQ,
   2135    LT, GT, UN), in fact we only need LEQ, and so we may as well
   2136    hardwire that fact.
   2137 
   2138    Returns zero iff LEQ(A,B), or a valid ThrID if not (zero is an
   2139    invald ThrID).  In the latter case, the returned ThrID indicates
   2140    the discovered point for which they are not.  There may be more
   2141    than one such point, but we only care about seeing one of them, not
   2142    all of them.  This rather strange convention is used because
   2143    sometimes we want to know the actual index at which they first
   2144    differ. */
   2145 static UInt VTS__cmpLEQ ( VTS* a, VTS* b );
   2146 
   2147 /* Compute an arbitrary structural (total) ordering on the two args,
   2148    based on their VCs, so they can be looked up in a table, tree, etc.
   2149    Returns -1, 0 or 1. */
   2150 static Word VTS__cmp_structural ( VTS* a, VTS* b );
   2151 
   2152 /* Debugging only.  Display the given VTS. */
   2153 static void VTS__show ( const VTS* vts );
   2154 
   2155 /* Debugging only.  Return vts[index], so to speak. */
   2156 static ULong VTS__indexAt_SLOW ( VTS* vts, Thr* idx );
   2157 
   2158 /* Notify the VTS machinery that a thread has been declared
   2159    comprehensively dead: that is, it has done an async exit AND it has
   2160    been joined with.  This should ensure that its local clocks (.viR
   2161    and .viW) will never again change, and so all mentions of this
   2162    thread from all VTSs in the system may be removed. */
   2163 static void VTS__declare_thread_very_dead ( Thr* idx );
   2164 
   2165 /*--------------- to do with Vector Timestamps ---------------*/
   2166 
   2167 static Bool is_sane_VTS ( VTS* vts )
   2168 {
   2169    UWord     i, n;
   2170    ScalarTS  *st1, *st2;
   2171    if (!vts) return False;
   2172    if (vts->usedTS > vts->sizeTS) return False;
   2173    n = vts->usedTS;
   2174    if (n == 1) {
   2175       st1 = &vts->ts[0];
   2176       if (st1->tym == 0)
   2177          return False;
   2178    }
   2179    else
   2180    if (n >= 2) {
   2181       for (i = 0; i < n-1; i++) {
   2182          st1 = &vts->ts[i];
   2183          st2 = &vts->ts[i+1];
   2184          if (st1->thrid >= st2->thrid)
   2185             return False;
   2186          if (st1->tym == 0 || st2->tym == 0)
   2187             return False;
   2188       }
   2189    }
   2190    return True;
   2191 }
   2192 
   2193 
   2194 /* Create a new, empty VTS.
   2195 */
   2196 static VTS* VTS__new ( const HChar* who, UInt sizeTS )
   2197 {
   2198    VTS* vts = HG_(zalloc)(who, sizeof(VTS) + (sizeTS+1) * sizeof(ScalarTS));
   2199    tl_assert(vts->usedTS == 0);
   2200    vts->sizeTS = sizeTS;
   2201    *(ULong*)(&vts->ts[sizeTS]) = 0x0ddC0ffeeBadF00dULL;
   2202    return vts;
   2203 }
   2204 
   2205 /* Clone this VTS.
   2206 */
   2207 static VTS* VTS__clone ( const HChar* who, VTS* vts )
   2208 {
   2209    tl_assert(vts);
   2210    tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL);
   2211    UInt nTS = vts->usedTS;
   2212    VTS* clone = VTS__new(who, nTS);
   2213    clone->id = vts->id;
   2214    clone->sizeTS = nTS;
   2215    clone->usedTS = nTS;
   2216    UInt i;
   2217    for (i = 0; i < nTS; i++) {
   2218       clone->ts[i] = vts->ts[i];
   2219    }
   2220    tl_assert( *(ULong*)(&clone->ts[clone->sizeTS]) == 0x0ddC0ffeeBadF00dULL);
   2221    return clone;
   2222 }
   2223 
   2224 
   2225 /* Make a clone of a VTS with specified ThrIDs removed.  'thridsToDel'
   2226    must be in strictly increasing order.  We could obviously do this
   2227    much more efficiently (in linear time) if necessary.
   2228 */
   2229 static VTS* VTS__subtract ( const HChar* who, VTS* vts, XArray* thridsToDel )
   2230 {
   2231    UInt i, j;
   2232    tl_assert(vts);
   2233    tl_assert(thridsToDel);
   2234    tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL);
   2235    UInt nTS = vts->usedTS;
   2236    /* Figure out how many ScalarTSs will remain in the output. */
   2237    UInt nReq = nTS;
   2238    for (i = 0; i < nTS; i++) {
   2239       ThrID thrid = vts->ts[i].thrid;
   2240       if (VG_(lookupXA)(thridsToDel, &thrid, NULL, NULL))
   2241          nReq--;
   2242    }
   2243    tl_assert(nReq <= nTS);
   2244    /* Copy the ones that will remain. */
   2245    VTS* res = VTS__new(who, nReq);
   2246    j = 0;
   2247    for (i = 0; i < nTS; i++) {
   2248       ThrID thrid = vts->ts[i].thrid;
   2249       if (VG_(lookupXA)(thridsToDel, &thrid, NULL, NULL))
   2250          continue;
   2251       res->ts[j++] = vts->ts[i];
   2252    }
   2253    tl_assert(j == nReq);
   2254    tl_assert(j == res->sizeTS);
   2255    res->usedTS = j;
   2256    tl_assert( *(ULong*)(&res->ts[j]) == 0x0ddC0ffeeBadF00dULL);
   2257    return res;
   2258 }
   2259 
   2260 
   2261 /* Delete this VTS in its entirety.
   2262 */
   2263 static void VTS__delete ( VTS* vts )
   2264 {
   2265    tl_assert(vts);
   2266    tl_assert(vts->usedTS <= vts->sizeTS);
   2267    tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL);
   2268    HG_(free)(vts);
   2269 }
   2270 
   2271 
   2272 /* Create a new singleton VTS.
   2273 */
   2274 static void VTS__singleton ( /*OUT*/VTS* out, Thr* thr, ULong tym )
   2275 {
   2276    tl_assert(thr);
   2277    tl_assert(tym >= 1);
   2278    tl_assert(out);
   2279    tl_assert(out->usedTS == 0);
   2280    tl_assert(out->sizeTS >= 1);
   2281    UInt hi = out->usedTS++;
   2282    out->ts[hi].thrid = Thr__to_ThrID(thr);
   2283    out->ts[hi].tym   = tym;
   2284 }
   2285 
   2286 
   2287 /* Return a new VTS in which vts[me]++, so to speak.  'vts' itself is
   2288    not modified.
   2289 */
   2290 static void VTS__tick ( /*OUT*/VTS* out, Thr* me, VTS* vts )
   2291 {
   2292    UInt      i, n;
   2293    ThrID     me_thrid;
   2294    Bool      found = False;
   2295 
   2296    stats__vts__tick++;
   2297 
   2298    tl_assert(out);
   2299    tl_assert(out->usedTS == 0);
   2300    if (vts->usedTS >= ThrID_MAX_VALID)
   2301       scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ );
   2302    tl_assert(out->sizeTS >= 1 + vts->usedTS);
   2303 
   2304    tl_assert(me);
   2305    me_thrid = Thr__to_ThrID(me);
   2306    tl_assert(is_sane_VTS(vts));
   2307    n = vts->usedTS;
   2308 
   2309    /* Copy all entries which precede 'me'. */
   2310    for (i = 0; i < n; i++) {
   2311       ScalarTS* here = &vts->ts[i];
   2312       if (UNLIKELY(here->thrid >= me_thrid))
   2313          break;
   2314       UInt hi = out->usedTS++;
   2315       out->ts[hi] = *here;
   2316    }
   2317 
   2318    /* 'i' now indicates the next entry to copy, if any.
   2319        There are 3 possibilities:
   2320        (a) there is no next entry (we used them all up already):
   2321            add (me_thrid,1) to the output, and quit
   2322        (b) there is a next entry, and its thrid > me_thrid:
   2323            add (me_thrid,1) to the output, then copy the remaining entries
   2324        (c) there is a next entry, and its thrid == me_thrid:
   2325            copy it to the output but increment its timestamp value.
   2326            Then copy the remaining entries.  (c) is the common case.
   2327    */
   2328    tl_assert(i >= 0 && i <= n);
   2329    if (i == n) { /* case (a) */
   2330       UInt hi = out->usedTS++;
   2331       out->ts[hi].thrid = me_thrid;
   2332       out->ts[hi].tym   = 1;
   2333    } else {
   2334       /* cases (b) and (c) */
   2335       ScalarTS* here = &vts->ts[i];
   2336       if (me_thrid == here->thrid) { /* case (c) */
   2337          if (UNLIKELY(here->tym >= (1ULL << SCALARTS_N_TYMBITS) - 2ULL)) {
   2338             /* We're hosed.  We have to stop. */
   2339             scalarts_limitations_fail_NORETURN( False/*!due_to_nThrs*/ );
   2340          }
   2341          UInt hi = out->usedTS++;
   2342          out->ts[hi].thrid = here->thrid;
   2343          out->ts[hi].tym   = here->tym + 1;
   2344          i++;
   2345          found = True;
   2346       } else { /* case (b) */
   2347          UInt hi = out->usedTS++;
   2348          out->ts[hi].thrid = me_thrid;
   2349          out->ts[hi].tym   = 1;
   2350       }
   2351       /* And copy any remaining entries. */
   2352       for (/*keepgoing*/; i < n; i++) {
   2353          ScalarTS* here2 = &vts->ts[i];
   2354          UInt hi = out->usedTS++;
   2355          out->ts[hi] = *here2;
   2356       }
   2357    }
   2358 
   2359    tl_assert(is_sane_VTS(out));
   2360    tl_assert(out->usedTS == vts->usedTS + (found ? 0 : 1));
   2361    tl_assert(out->usedTS <= out->sizeTS);
   2362 }
   2363 
   2364 
   2365 /* Return a new VTS constructed as the join (max) of the 2 args.
   2366    Neither arg is modified.
   2367 */
   2368 static void VTS__join ( /*OUT*/VTS* out, VTS* a, VTS* b )
   2369 {
   2370    UInt     ia, ib, useda, usedb;
   2371    ULong    tyma, tymb, tymMax;
   2372    ThrID    thrid;
   2373    UInt     ncommon = 0;
   2374 
   2375    stats__vts__join++;
   2376 
   2377    tl_assert(a);
   2378    tl_assert(b);
   2379    useda = a->usedTS;
   2380    usedb = b->usedTS;
   2381 
   2382    tl_assert(out);
   2383    tl_assert(out->usedTS == 0);
   2384    /* overly conservative test, but doing better involves comparing
   2385       the two VTSs, which we don't want to do at this point. */
   2386    if (useda + usedb >= ThrID_MAX_VALID)
   2387       scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ );
   2388    tl_assert(out->sizeTS >= useda + usedb);
   2389 
   2390    ia = ib = 0;
   2391 
   2392    while (1) {
   2393 
   2394       /* This logic is to enumerate triples (thrid, tyma, tymb) drawn
   2395          from a and b in order, where thrid is the next ThrID
   2396          occurring in either a or b, and tyma/b are the relevant
   2397          scalar timestamps, taking into account implicit zeroes. */
   2398       tl_assert(ia >= 0 && ia <= useda);
   2399       tl_assert(ib >= 0 && ib <= usedb);
   2400 
   2401       if        (ia == useda && ib == usedb) {
   2402          /* both empty - done */
   2403          break;
   2404 
   2405       } else if (ia == useda && ib != usedb) {
   2406          /* a empty, use up b */
   2407          ScalarTS* tmpb = &b->ts[ib];
   2408          thrid = tmpb->thrid;
   2409          tyma  = 0;
   2410          tymb  = tmpb->tym;
   2411          ib++;
   2412 
   2413       } else if (ia != useda && ib == usedb) {
   2414          /* b empty, use up a */
   2415          ScalarTS* tmpa = &a->ts[ia];
   2416          thrid = tmpa->thrid;
   2417          tyma  = tmpa->tym;
   2418          tymb  = 0;
   2419          ia++;
   2420 
   2421       } else {
   2422          /* both not empty; extract lowest-ThrID'd triple */
   2423          ScalarTS* tmpa = &a->ts[ia];
   2424          ScalarTS* tmpb = &b->ts[ib];
   2425          if (tmpa->thrid < tmpb->thrid) {
   2426             /* a has the lowest unconsidered ThrID */
   2427             thrid = tmpa->thrid;
   2428             tyma  = tmpa->tym;
   2429             tymb  = 0;
   2430             ia++;
   2431          } else if (tmpa->thrid > tmpb->thrid) {
   2432             /* b has the lowest unconsidered ThrID */
   2433             thrid = tmpb->thrid;
   2434             tyma  = 0;
   2435             tymb  = tmpb->tym;
   2436             ib++;
   2437          } else {
   2438             /* they both next mention the same ThrID */
   2439             tl_assert(tmpa->thrid == tmpb->thrid);
   2440             thrid = tmpa->thrid; /* == tmpb->thrid */
   2441             tyma  = tmpa->tym;
   2442             tymb  = tmpb->tym;
   2443             ia++;
   2444             ib++;
   2445             ncommon++;
   2446          }
   2447       }
   2448 
   2449       /* having laboriously determined (thr, tyma, tymb), do something
   2450          useful with it. */
   2451       tymMax = tyma > tymb ? tyma : tymb;
   2452       if (tymMax > 0) {
   2453          UInt hi = out->usedTS++;
   2454          out->ts[hi].thrid = thrid;
   2455          out->ts[hi].tym   = tymMax;
   2456       }
   2457 
   2458    }
   2459 
   2460    tl_assert(is_sane_VTS(out));
   2461    tl_assert(out->usedTS <= out->sizeTS);
   2462    tl_assert(out->usedTS == useda + usedb - ncommon);
   2463 }
   2464 
   2465 
   2466 /* Determine if 'a' <= 'b', in the partial ordering.  Returns zero if
   2467    they are, or the first ThrID for which they are not (no valid ThrID
   2468    has the value zero).  This rather strange convention is used
   2469    because sometimes we want to know the actual index at which they
   2470    first differ. */
   2471 static UInt/*ThrID*/ VTS__cmpLEQ ( VTS* a, VTS* b )
   2472 {
   2473    Word  ia, ib, useda, usedb;
   2474    ULong tyma, tymb;
   2475 
   2476    stats__vts__cmpLEQ++;
   2477 
   2478    tl_assert(a);
   2479    tl_assert(b);
   2480    useda = a->usedTS;
   2481    usedb = b->usedTS;
   2482 
   2483    ia = ib = 0;
   2484 
   2485    while (1) {
   2486 
   2487       /* This logic is to enumerate doubles (tyma, tymb) drawn
   2488          from a and b in order, and tyma/b are the relevant
   2489          scalar timestamps, taking into account implicit zeroes. */
   2490       ThrID thrid;
   2491 
   2492       tl_assert(ia >= 0 && ia <= useda);
   2493       tl_assert(ib >= 0 && ib <= usedb);
   2494 
   2495       if        (ia == useda && ib == usedb) {
   2496          /* both empty - done */
   2497          break;
   2498 
   2499       } else if (ia == useda && ib != usedb) {
   2500          /* a empty, use up b */
   2501          ScalarTS* tmpb = &b->ts[ib];
   2502          tyma  = 0;
   2503          tymb  = tmpb->tym;
   2504          thrid = tmpb->thrid;
   2505          ib++;
   2506 
   2507       } else if (ia != useda && ib == usedb) {
   2508          /* b empty, use up a */
   2509          ScalarTS* tmpa = &a->ts[ia];
   2510          tyma  = tmpa->tym;
   2511          thrid = tmpa->thrid;
   2512          tymb  = 0;
   2513          ia++;
   2514 
   2515       } else {
   2516          /* both not empty; extract lowest-ThrID'd triple */
   2517          ScalarTS* tmpa = &a->ts[ia];
   2518          ScalarTS* tmpb = &b->ts[ib];
   2519          if (tmpa->thrid < tmpb->thrid) {
   2520             /* a has the lowest unconsidered ThrID */
   2521             tyma  = tmpa->tym;
   2522             thrid = tmpa->thrid;
   2523             tymb  = 0;
   2524             ia++;
   2525          }
   2526          else
   2527          if (tmpa->thrid > tmpb->thrid) {
   2528             /* b has the lowest unconsidered ThrID */
   2529             tyma  = 0;
   2530             tymb  = tmpb->tym;
   2531             thrid = tmpb->thrid;
   2532             ib++;
   2533          } else {
   2534             /* they both next mention the same ThrID */
   2535             tl_assert(tmpa->thrid == tmpb->thrid);
   2536             tyma  = tmpa->tym;
   2537             thrid = tmpa->thrid;
   2538             tymb  = tmpb->tym;
   2539             ia++;
   2540             ib++;
   2541          }
   2542       }
   2543 
   2544       /* having laboriously determined (tyma, tymb), do something
   2545          useful with it. */
   2546       if (tyma > tymb) {
   2547          /* not LEQ at this index.  Quit, since the answer is
   2548             determined already. */
   2549          tl_assert(thrid >= 1024);
   2550          return thrid;
   2551       }
   2552    }
   2553 
   2554    return 0; /* all points are LEQ => return an invalid ThrID */
   2555 }
   2556 
   2557 
   2558 /* Compute an arbitrary structural (total) ordering on the two args,
   2559    based on their VCs, so they can be looked up in a table, tree, etc.
   2560    Returns -1, 0 or 1.  (really just 'deriving Ord' :-) This can be
   2561    performance critical so there is some effort expended to make it sa
   2562    fast as possible.
   2563 */
   2564 Word VTS__cmp_structural ( VTS* a, VTS* b )
   2565 {
   2566    /* We just need to generate an arbitrary total ordering based on
   2567       a->ts and b->ts.  Preferably do it in a way which comes across likely
   2568       differences relatively quickly. */
   2569    Word     i;
   2570    Word     useda = 0,    usedb = 0;
   2571    ScalarTS *ctsa = NULL, *ctsb = NULL;
   2572 
   2573    stats__vts__cmp_structural++;
   2574 
   2575    tl_assert(a);
   2576    tl_assert(b);
   2577 
   2578    ctsa = &a->ts[0]; useda = a->usedTS;
   2579    ctsb = &b->ts[0]; usedb = b->usedTS;
   2580 
   2581    if (LIKELY(useda == usedb)) {
   2582       ScalarTS *tmpa = NULL, *tmpb = NULL;
   2583       stats__vts__cmp_structural_slow++;
   2584       /* Same length vectors.  Find the first difference, if any, as
   2585          fast as possible. */
   2586       for (i = 0; i < useda; i++) {
   2587          tmpa = &ctsa[i];
   2588          tmpb = &ctsb[i];
   2589          if (LIKELY(tmpa->tym == tmpb->tym
   2590                     && tmpa->thrid == tmpb->thrid))
   2591             continue;
   2592          else
   2593             break;
   2594       }
   2595       if (UNLIKELY(i == useda)) {
   2596          /* They're identical. */
   2597          return 0;
   2598       } else {
   2599          tl_assert(i >= 0 && i < useda);
   2600          if (tmpa->tym < tmpb->tym) return -1;
   2601          if (tmpa->tym > tmpb->tym) return 1;
   2602          if (tmpa->thrid < tmpb->thrid) return -1;
   2603          if (tmpa->thrid > tmpb->thrid) return 1;
   2604          /* we just established them as non-identical, hence: */
   2605       }
   2606       /*NOTREACHED*/
   2607       tl_assert(0);
   2608    }
   2609 
   2610    if (useda < usedb) return -1;
   2611    if (useda > usedb) return 1;
   2612    /*NOTREACHED*/
   2613    tl_assert(0);
   2614 }
   2615 
   2616 
   2617 /* Debugging only.  Display the given VTS.
   2618 */
   2619 static void VTS__show ( const VTS* vts )
   2620 {
   2621    Word      i, n;
   2622    tl_assert(vts);
   2623 
   2624    VG_(printf)("[");
   2625    n =  vts->usedTS;
   2626    for (i = 0; i < n; i++) {
   2627       const ScalarTS *st = &vts->ts[i];
   2628       VG_(printf)(i < n-1 ? "%d:%llu " : "%d:%llu", st->thrid, (ULong)st->tym);
   2629    }
   2630    VG_(printf)("]");
   2631 }
   2632 
   2633 
   2634 /* Debugging only.  Return vts[index], so to speak.
   2635 */
   2636 ULong VTS__indexAt_SLOW ( VTS* vts, Thr* idx )
   2637 {
   2638    UWord i, n;
   2639    ThrID idx_thrid = Thr__to_ThrID(idx);
   2640    stats__vts__indexat_slow++;
   2641    tl_assert(vts);
   2642    n = vts->usedTS;
   2643    for (i = 0; i < n; i++) {
   2644       ScalarTS* st = &vts->ts[i];
   2645       if (st->thrid == idx_thrid)
   2646          return st->tym;
   2647    }
   2648    return 0;
   2649 }
   2650 
   2651 
   2652 /* See comment on prototype above.
   2653 */
   2654 static void VTS__declare_thread_very_dead ( Thr* thr )
   2655 {
   2656    if (0) VG_(printf)("VTQ:  tae %p\n", thr);
   2657 
   2658    tl_assert(thr->llexit_done);
   2659    tl_assert(thr->joinedwith_done);
   2660 
   2661    ThrID nyu;
   2662    nyu = Thr__to_ThrID(thr);
   2663    VG_(addToXA)( verydead_thread_table_not_pruned, &nyu );
   2664 
   2665    /* We can only get here if we're assured that we'll never again
   2666       need to look at this thread's ::viR or ::viW.  Set them to
   2667       VtsID_INVALID, partly so as to avoid holding on to the VTSs, but
   2668       mostly so that we don't wind up pruning them (as that would be
   2669       nonsensical: the only interesting ScalarTS entry for a dead
   2670       thread is its own index, and the pruning will remove that.). */
   2671    VtsID__rcdec(thr->viR);
   2672    VtsID__rcdec(thr->viW);
   2673    thr->viR = VtsID_INVALID;
   2674    thr->viW = VtsID_INVALID;
   2675 }
   2676 
   2677 
   2678 /////////////////////////////////////////////////////////////////
   2679 /////////////////////////////////////////////////////////////////
   2680 //                                                             //
   2681 // SECTION END vts primitives                                  //
   2682 //                                                             //
   2683 /////////////////////////////////////////////////////////////////
   2684 /////////////////////////////////////////////////////////////////
   2685 
   2686 
   2687 
   2688 /////////////////////////////////////////////////////////////////
   2689 /////////////////////////////////////////////////////////////////
   2690 //                                                             //
   2691 // SECTION BEGIN main library                                  //
   2692 //                                                             //
   2693 /////////////////////////////////////////////////////////////////
   2694 /////////////////////////////////////////////////////////////////
   2695 
   2696 
   2697 /////////////////////////////////////////////////////////
   2698 //                                                     //
   2699 // VTS set                                             //
   2700 //                                                     //
   2701 /////////////////////////////////////////////////////////
   2702 
   2703 static WordFM* /* WordFM VTS* void */ vts_set = NULL;
   2704 
   2705 static void vts_set_init ( void )
   2706 {
   2707    tl_assert(!vts_set);
   2708    vts_set = VG_(newFM)( HG_(zalloc), "libhb.vts_set_init.1",
   2709                          HG_(free),
   2710                          (Word(*)(UWord,UWord))VTS__cmp_structural );
   2711 }
   2712 
   2713 /* Given a VTS, look in vts_set to see if we already have a
   2714    structurally identical one.  If yes, return the pair (True, pointer
   2715    to the existing one).  If no, clone this one, add the clone to the
   2716    set, and return (False, pointer to the clone). */
   2717 static Bool vts_set__find__or__clone_and_add ( /*OUT*/VTS** res, VTS* cand )
   2718 {
   2719    UWord keyW, valW;
   2720    stats__vts_set__focaa++;
   2721    tl_assert(cand->id == VtsID_INVALID);
   2722    /* lookup cand (by value) */
   2723    if (VG_(lookupFM)( vts_set, &keyW, &valW, (UWord)cand )) {
   2724       /* found it */
   2725       tl_assert(valW == 0);
   2726       /* if this fails, cand (by ref) was already present (!) */
   2727       tl_assert(keyW != (UWord)cand);
   2728       *res = (VTS*)keyW;
   2729       return True;
   2730    } else {
   2731       /* not present.  Clone, add and return address of clone. */
   2732       stats__vts_set__focaa_a++;
   2733       VTS* clone = VTS__clone( "libhb.vts_set_focaa.1", cand );
   2734       tl_assert(clone != cand);
   2735       VG_(addToFM)( vts_set, (UWord)clone, 0/*val is unused*/ );
   2736       *res = clone;
   2737       return False;
   2738    }
   2739 }
   2740 
   2741 
   2742 /////////////////////////////////////////////////////////
   2743 //                                                     //
   2744 // VTS table                                           //
   2745 //                                                     //
   2746 /////////////////////////////////////////////////////////
   2747 
   2748 static void VtsID__invalidate_caches ( void ); /* fwds */
   2749 
   2750 /* A type to hold VTS table entries.  Invariants:
   2751    If .vts == NULL, then this entry is not in use, so:
   2752    - .rc == 0
   2753    - this entry is on the freelist (unfortunately, does not imply
   2754      any constraints on value for u.freelink)
   2755    If .vts != NULL, then this entry is in use:
   2756    - .vts is findable in vts_set
   2757    - .vts->id == this entry number
   2758    - no specific value for .rc (even 0 is OK)
   2759    - this entry is not on freelist, so u.freelink == VtsID_INVALID
   2760 */
   2761 typedef
   2762    struct {
   2763       VTS*  vts;      /* vts, in vts_set */
   2764       UWord rc;       /* reference count - enough for entire aspace */
   2765       union {
   2766          VtsID freelink; /* chain for free entries, VtsID_INVALID at end */
   2767          VtsID remap;    /* used only during pruning, for used entries */
   2768       } u;
   2769       /* u.freelink only used when vts == NULL,
   2770          u.remap only used when vts != NULL, during pruning. */
   2771    }
   2772    VtsTE;
   2773 
   2774 /* The VTS table. */
   2775 static XArray* /* of VtsTE */ vts_tab = NULL;
   2776 
   2777 /* An index into the VTS table, indicating the start of the list of
   2778    free (available for use) entries.  If the list is empty, this is
   2779    VtsID_INVALID. */
   2780 static VtsID vts_tab_freelist = VtsID_INVALID;
   2781 
   2782 /* Do a GC of vts_tab when the freelist becomes empty AND the size of
   2783    vts_tab equals or exceeds this size.  After GC, the value here is
   2784    set appropriately so as to check for the next GC point. */
   2785 static Word vts_next_GC_at = 1000;
   2786 
   2787 static void vts_tab_init ( void )
   2788 {
   2789    vts_tab = VG_(newXA)( HG_(zalloc), "libhb.vts_tab_init.1",
   2790                          HG_(free), sizeof(VtsTE) );
   2791    vts_tab_freelist = VtsID_INVALID;
   2792 }
   2793 
   2794 /* Add ii to the free list, checking that it looks out-of-use. */
   2795 static void add_to_free_list ( VtsID ii )
   2796 {
   2797    VtsTE* ie = VG_(indexXA)( vts_tab, ii );
   2798    tl_assert(ie->vts == NULL);
   2799    tl_assert(ie->rc == 0);
   2800    tl_assert(ie->u.freelink == VtsID_INVALID);
   2801    ie->u.freelink = vts_tab_freelist;
   2802    vts_tab_freelist = ii;
   2803 }
   2804 
   2805 /* Get an entry from the free list.  This will return VtsID_INVALID if
   2806    the free list is empty. */
   2807 static VtsID get_from_free_list ( void )
   2808 {
   2809    VtsID  ii;
   2810    VtsTE* ie;
   2811    if (vts_tab_freelist == VtsID_INVALID)
   2812       return VtsID_INVALID;
   2813    ii = vts_tab_freelist;
   2814    ie = VG_(indexXA)( vts_tab, ii );
   2815    tl_assert(ie->vts == NULL);
   2816    tl_assert(ie->rc == 0);
   2817    vts_tab_freelist = ie->u.freelink;
   2818    return ii;
   2819 }
   2820 
   2821 /* Produce a new VtsID that can be used, either by getting it from
   2822    the freelist, or, if that is empty, by expanding vts_tab. */
   2823 static VtsID get_new_VtsID ( void )
   2824 {
   2825    VtsID ii;
   2826    VtsTE te;
   2827    ii = get_from_free_list();
   2828    if (ii != VtsID_INVALID)
   2829       return ii;
   2830    te.vts = NULL;
   2831    te.rc = 0;
   2832    te.u.freelink = VtsID_INVALID;
   2833    ii = (VtsID)VG_(addToXA)( vts_tab, &te );
   2834    return ii;
   2835 }
   2836 
   2837 
   2838 /* Indirect callback from lib_zsm. */
   2839 static void VtsID__rcinc ( VtsID ii )
   2840 {
   2841    VtsTE* ie;
   2842    /* VG_(indexXA) does a range check for us */
   2843    ie = VG_(indexXA)( vts_tab, ii );
   2844    tl_assert(ie->vts); /* else it's not in use */
   2845    tl_assert(ie->rc < ~0UL); /* else we can't continue */
   2846    tl_assert(ie->vts->id == ii);
   2847    ie->rc++;
   2848 }
   2849 
   2850 /* Indirect callback from lib_zsm. */
   2851 static void VtsID__rcdec ( VtsID ii )
   2852 {
   2853    VtsTE* ie;
   2854    /* VG_(indexXA) does a range check for us */
   2855    ie = VG_(indexXA)( vts_tab, ii );
   2856    tl_assert(ie->vts); /* else it's not in use */
   2857    tl_assert(ie->rc > 0); /* else RC snafu */
   2858    tl_assert(ie->vts->id == ii);
   2859    ie->rc--;
   2860 }
   2861 
   2862 
   2863 /* Look up 'cand' in our collection of VTSs.  If present, return the
   2864    VtsID for the pre-existing version.  If not present, clone it, add
   2865    the clone to both vts_tab and vts_set, allocate a fresh VtsID for
   2866    it, and return that. */
   2867 static VtsID vts_tab__find__or__clone_and_add ( VTS* cand )
   2868 {
   2869    VTS* in_tab = NULL;
   2870    tl_assert(cand->id == VtsID_INVALID);
   2871    Bool already_have = vts_set__find__or__clone_and_add( &in_tab, cand );
   2872    tl_assert(in_tab);
   2873    if (already_have) {
   2874       /* We already have a copy of 'cand'.  Use that. */
   2875       VtsTE* ie;
   2876       tl_assert(in_tab->id != VtsID_INVALID);
   2877       ie = VG_(indexXA)( vts_tab, in_tab->id );
   2878       tl_assert(ie->vts == in_tab);
   2879       return in_tab->id;
   2880    } else {
   2881       VtsID  ii = get_new_VtsID();
   2882       VtsTE* ie = VG_(indexXA)( vts_tab, ii );
   2883       ie->vts = in_tab;
   2884       ie->rc = 0;
   2885       ie->u.freelink = VtsID_INVALID;
   2886       in_tab->id = ii;
   2887       return ii;
   2888    }
   2889 }
   2890 
   2891 
   2892 static void show_vts_stats ( const HChar* caller )
   2893 {
   2894    UWord nSet, nTab, nLive;
   2895    ULong totrc;
   2896    UWord n, i;
   2897    nSet = VG_(sizeFM)( vts_set );
   2898    nTab = VG_(sizeXA)( vts_tab );
   2899    totrc = 0;
   2900    nLive = 0;
   2901    n = VG_(sizeXA)( vts_tab );
   2902    for (i = 0; i < n; i++) {
   2903       VtsTE* ie = VG_(indexXA)( vts_tab, i );
   2904       if (ie->vts) {
   2905          nLive++;
   2906          totrc += (ULong)ie->rc;
   2907       } else {
   2908          tl_assert(ie->rc == 0);
   2909       }
   2910    }
   2911    VG_(printf)("  show_vts_stats %s\n", caller);
   2912    VG_(printf)("    vts_tab size %4lu\n", nTab);
   2913    VG_(printf)("    vts_tab live %4lu\n", nLive);
   2914    VG_(printf)("    vts_set size %4lu\n", nSet);
   2915    VG_(printf)("        total rc %4llu\n", totrc);
   2916 }
   2917 
   2918 
   2919 /* --- Helpers for VtsID pruning --- */
   2920 
   2921 static
   2922 void remap_VtsID ( /*MOD*/XArray* /* of VtsTE */ old_tab,
   2923                    /*MOD*/XArray* /* of VtsTE */ new_tab,
   2924                    VtsID* ii )
   2925 {
   2926    VtsTE *old_te, *new_te;
   2927    VtsID old_id, new_id;
   2928    /* We're relying here on VG_(indexXA)'s range checking to assert on
   2929       any stupid values, in particular *ii == VtsID_INVALID. */
   2930    old_id = *ii;
   2931    old_te = VG_(indexXA)( old_tab, old_id );
   2932    old_te->rc--;
   2933    new_id = old_te->u.remap;
   2934    new_te = VG_(indexXA)( new_tab, new_id );
   2935    new_te->rc++;
   2936    *ii = new_id;
   2937 }
   2938 
   2939 static
   2940 void remap_VtsIDs_in_SVal ( /*MOD*/XArray* /* of VtsTE */ old_tab,
   2941                             /*MOD*/XArray* /* of VtsTE */ new_tab,
   2942                             SVal* s )
   2943 {
   2944    SVal old_sv, new_sv;
   2945    old_sv = *s;
   2946    if (SVal__isC(old_sv)) {
   2947       VtsID rMin, wMin;
   2948       rMin = SVal__unC_Rmin(old_sv);
   2949       wMin = SVal__unC_Wmin(old_sv);
   2950       remap_VtsID( old_tab, new_tab, &rMin );
   2951       remap_VtsID( old_tab, new_tab, &wMin );
   2952       new_sv = SVal__mkC( rMin, wMin );
   2953       *s = new_sv;
   2954   }
   2955 }
   2956 
   2957 
   2958 /* NOT TO BE CALLED FROM WITHIN libzsm. */
   2959 __attribute__((noinline))
   2960 static void vts_tab__do_GC ( Bool show_stats )
   2961 {
   2962    UWord i, nTab, nLive, nFreed;
   2963 
   2964    /* ---------- BEGIN VTS GC ---------- */
   2965    /* check this is actually necessary. */
   2966    tl_assert(vts_tab_freelist == VtsID_INVALID);
   2967 
   2968    /* empty the caches for partial order checks and binary joins.  We
   2969       could do better and prune out the entries to be deleted, but it
   2970       ain't worth the hassle. */
   2971    VtsID__invalidate_caches();
   2972 
   2973    /* First, make the reference counts up to date. */
   2974    zsm_flush_cache();
   2975 
   2976    nTab = VG_(sizeXA)( vts_tab );
   2977 
   2978    if (show_stats) {
   2979       VG_(printf)("<<GC begins at vts_tab size %lu>>\n", nTab);
   2980       show_vts_stats("before GC");
   2981    }
   2982 
   2983    /* Now we can inspect the entire vts_tab.  Any entries with zero
   2984       .rc fields are now no longer in use and can be put back on the
   2985       free list, removed from vts_set, and deleted. */
   2986    nFreed = 0;
   2987    for (i = 0; i < nTab; i++) {
   2988       Bool present;
   2989       UWord oldK = 0, oldV = 12345;
   2990       VtsTE* te = VG_(indexXA)( vts_tab, i );
   2991       if (te->vts == NULL) {
   2992          tl_assert(te->rc == 0);
   2993          continue; /* already on the free list (presumably) */
   2994       }
   2995       if (te->rc > 0)
   2996          continue; /* in use */
   2997       /* Ok, we got one we can free. */
   2998       tl_assert(te->vts->id == i);
   2999       /* first, remove it from vts_set. */
   3000       present = VG_(delFromFM)( vts_set,
   3001                                 &oldK, &oldV, (UWord)te->vts );
   3002       tl_assert(present); /* else it isn't in vts_set ?! */
   3003       tl_assert(oldV == 0); /* no info stored in vts_set val fields */
   3004       tl_assert(oldK == (UWord)te->vts); /* else what did delFromFM find?! */
   3005       /* now free the VTS itself */
   3006       VTS__delete(te->vts);
   3007       te->vts = NULL;
   3008       /* and finally put this entry on the free list */
   3009       tl_assert(te->u.freelink == VtsID_INVALID); /* can't already be on it */
   3010       add_to_free_list( i );
   3011       nFreed++;
   3012    }
   3013 
   3014    /* Now figure out when the next GC should be.  We'll allow the
   3015       number of VTSs to double before GCing again.  Except of course
   3016       that since we can't (or, at least, don't) shrink vts_tab, we
   3017       can't set the threshold value smaller than it. */
   3018    tl_assert(nFreed <= nTab);
   3019    nLive = nTab - nFreed;
   3020    tl_assert(nLive >= 0 && nLive <= nTab);
   3021    vts_next_GC_at = 2 * nLive;
   3022    if (vts_next_GC_at < nTab)
   3023       vts_next_GC_at = nTab;
   3024 
   3025    if (show_stats) {
   3026       show_vts_stats("after GC");
   3027       VG_(printf)("<<GC ends, next gc at %ld>>\n", vts_next_GC_at);
   3028    }
   3029 
   3030    stats__vts_tab_GC++;
   3031    if (VG_(clo_stats)) {
   3032       tl_assert(nTab > 0);
   3033       VG_(message)(Vg_DebugMsg,
   3034                    "libhb: VTS GC: #%lu  old size %lu  live %lu  (%2llu%%)\n",
   3035                    stats__vts_tab_GC,
   3036                    nTab, nLive, (100ULL * (ULong)nLive) / (ULong)nTab);
   3037    }
   3038    /* ---------- END VTS GC ---------- */
   3039 
   3040    /* Decide whether to do VTS pruning.  We have one of three
   3041       settings. */
   3042    static UInt pruning_auto_ctr = 0; /* do not make non-static */
   3043 
   3044    Bool do_pruning = False;
   3045    switch (HG_(clo_vts_pruning)) {
   3046       case 0: /* never */
   3047          break;
   3048       case 1: /* auto */
   3049          do_pruning = (++pruning_auto_ctr % 5) == 0;
   3050          break;
   3051       case 2: /* always */
   3052          do_pruning = True;
   3053          break;
   3054       default:
   3055          tl_assert(0);
   3056    }
   3057 
   3058    /* The rest of this routine only handles pruning, so we can
   3059       quit at this point if it is not to be done. */
   3060    if (!do_pruning)
   3061       return;
   3062    /* No need to do pruning if no thread died since the last pruning as
   3063       no VtsTE can be pruned. */
   3064    if (VG_(sizeXA)( verydead_thread_table_not_pruned) == 0)
   3065       return;
   3066 
   3067    /* ---------- BEGIN VTS PRUNING ---------- */
   3068    /* Sort and check the very dead threads that died since the last pruning.
   3069       Sorting is used for the check and so that we can quickly look
   3070       up the dead-thread entries as we work through the VTSs. */
   3071    verydead_thread_table_sort_and_check (verydead_thread_table_not_pruned);
   3072 
   3073    /* We will run through the old table, and create a new table and
   3074       set, at the same time setting the u.remap entries in the old
   3075       table to point to the new entries.  Then, visit every VtsID in
   3076       the system, and replace all of them with new ones, using the
   3077       u.remap entries in the old table.  Finally, we can delete the old
   3078       table and set. */
   3079 
   3080    XArray* /* of VtsTE */ new_tab
   3081       = VG_(newXA)( HG_(zalloc), "libhb.vts_tab__do_GC.new_tab",
   3082                     HG_(free), sizeof(VtsTE) );
   3083 
   3084    /* WordFM VTS* void */
   3085    WordFM* new_set
   3086       = VG_(newFM)( HG_(zalloc), "libhb.vts_tab__do_GC.new_set",
   3087                     HG_(free),
   3088                     (Word(*)(UWord,UWord))VTS__cmp_structural );
   3089 
   3090    /* Visit each old VTS.  For each one:
   3091 
   3092       * make a pruned version
   3093 
   3094       * search new_set for the pruned version, yielding either
   3095         Nothing (not present) or the new VtsID for it.
   3096 
   3097       * if not present, allocate a new VtsID for it, insert (pruned
   3098         VTS, new VtsID) in the tree, and set
   3099         remap_table[old VtsID] = new VtsID.
   3100 
   3101       * if present, set remap_table[old VtsID] = new VtsID, where
   3102         new VtsID was determined by the tree lookup.  Then free up
   3103         the clone.
   3104    */
   3105 
   3106    UWord nBeforePruning = 0, nAfterPruning = 0;
   3107    UWord nSTSsBefore = 0, nSTSsAfter = 0;
   3108    VtsID new_VtsID_ctr = 0;
   3109 
   3110    for (i = 0; i < nTab; i++) {
   3111 
   3112       /* For each old VTS .. */
   3113       VtsTE* old_te  = VG_(indexXA)( vts_tab, i );
   3114       VTS*   old_vts = old_te->vts;
   3115 
   3116       /* Skip it if not in use */
   3117       if (old_te->rc == 0) {
   3118          tl_assert(old_vts == NULL);
   3119          continue;
   3120       }
   3121       tl_assert(old_te->u.remap == VtsID_INVALID);
   3122       tl_assert(old_vts != NULL);
   3123       tl_assert(old_vts->id == i);
   3124       tl_assert(old_vts->ts != NULL);
   3125 
   3126       /* It is in use. Make a pruned version. */
   3127       nBeforePruning++;
   3128       nSTSsBefore += old_vts->usedTS;
   3129       VTS* new_vts = VTS__subtract("libhb.vts_tab__do_GC.new_vts",
   3130                                    old_vts, verydead_thread_table_not_pruned);
   3131       tl_assert(new_vts->sizeTS == new_vts->usedTS);
   3132       tl_assert(*(ULong*)(&new_vts->ts[new_vts->usedTS])
   3133                 == 0x0ddC0ffeeBadF00dULL);
   3134 
   3135       /* Get rid of the old VTS and the tree entry.  It's a bit more
   3136          complex to incrementally delete the VTSs now than to nuke
   3137          them all after we're done, but the upside is that we don't
   3138          wind up temporarily storing potentially two complete copies
   3139          of each VTS and hence spiking memory use. */
   3140       UWord oldK = 0, oldV = 12345;
   3141       Bool  present = VG_(delFromFM)( vts_set,
   3142                                       &oldK, &oldV, (UWord)old_vts );
   3143       tl_assert(present); /* else it isn't in vts_set ?! */
   3144       tl_assert(oldV == 0); /* no info stored in vts_set val fields */
   3145       tl_assert(oldK == (UWord)old_vts); /* else what did delFromFM find?! */
   3146       /* now free the VTS itself */
   3147       VTS__delete(old_vts);
   3148       old_te->vts = NULL;
   3149       old_vts = NULL;
   3150 
   3151       /* NO MENTIONS of old_vts allowed beyond this point. */
   3152 
   3153       /* Ok, we have the pruned copy in new_vts.  See if a
   3154          structurally identical version is already present in new_set.
   3155          If so, delete the one we just made and move on; if not, add
   3156          it. */
   3157       VTS*  identical_version = NULL;
   3158       UWord valW = 12345;
   3159       if (VG_(lookupFM)(new_set, (UWord*)&identical_version, &valW,
   3160                         (UWord)new_vts)) {
   3161          // already have it
   3162          tl_assert(valW == 0);
   3163          tl_assert(identical_version != NULL);
   3164          tl_assert(identical_version != new_vts);
   3165          VTS__delete(new_vts);
   3166          new_vts = identical_version;
   3167          tl_assert(new_vts->id != VtsID_INVALID);
   3168       } else {
   3169          tl_assert(valW == 12345);
   3170          tl_assert(identical_version == NULL);
   3171          new_vts->id = new_VtsID_ctr++;
   3172          Bool b = VG_(addToFM)(new_set, (UWord)new_vts, 0);
   3173          tl_assert(!b);
   3174          VtsTE new_te;
   3175          new_te.vts      = new_vts;
   3176          new_te.rc       = 0;
   3177          new_te.u.freelink = VtsID_INVALID;
   3178          Word j = VG_(addToXA)( new_tab, &new_te );
   3179          tl_assert(j <= i);
   3180          tl_assert(j == new_VtsID_ctr - 1);
   3181          // stats
   3182          nAfterPruning++;
   3183          nSTSsAfter += new_vts->usedTS;
   3184       }
   3185       old_te->u.remap = new_vts->id;
   3186 
   3187    } /* for (i = 0; i < nTab; i++) */
   3188 
   3189    /* Move very dead thread from verydead_thread_table_not_pruned to
   3190       verydead_thread_table. Sort and check verydead_thread_table
   3191       to verify a thread was reported very dead only once. */
   3192    {
   3193       UWord nBT = VG_(sizeXA)( verydead_thread_table_not_pruned);
   3194 
   3195       for (i = 0; i < nBT; i++) {
   3196          ThrID thrid =
   3197             *(ThrID*)VG_(indexXA)( verydead_thread_table_not_pruned, i );
   3198          VG_(addToXA)( verydead_thread_table, &thrid );
   3199       }
   3200       verydead_thread_table_sort_and_check (verydead_thread_table);
   3201       VG_(dropHeadXA) (verydead_thread_table_not_pruned, nBT);
   3202    }
   3203 
   3204    /* At this point, we have:
   3205       * the old VTS table, with its u.remap entries set,
   3206         and with all .vts == NULL.
   3207       * the old VTS tree should be empty, since it and the old VTSs
   3208         it contained have been incrementally deleted was we worked
   3209         through the old table.
   3210       * the new VTS table, with all .rc == 0, all u.freelink and u.remap
   3211         == VtsID_INVALID.
   3212       * the new VTS tree.
   3213    */
   3214    tl_assert( VG_(sizeFM)(vts_set) == 0 );
   3215 
   3216    /* Now actually apply the mapping. */
   3217    /* Visit all the VtsIDs in the entire system.  Where do we expect
   3218       to find them?
   3219       (a) in shadow memory -- the LineZs and LineFs
   3220       (b) in our collection of struct _Thrs.
   3221       (c) in our collection of struct _SOs.
   3222       Nowhere else, AFAICS.  Not in the zsm cache, because that just
   3223       got invalidated.
   3224 
   3225       Using the u.remap fields in vts_tab, map each old VtsID to a new
   3226       VtsID.  For each old VtsID, dec its rc; and for each new one,
   3227       inc it.  This sets up the new refcounts, and it also gives a
   3228       cheap sanity check of the old ones: all old refcounts should be
   3229       zero after this operation.
   3230    */
   3231 
   3232    /* Do the mappings for (a) above: iterate over the Primary shadow
   3233       mem map (WordFM Addr SecMap*). */
   3234    UWord secmapW = 0;
   3235    VG_(initIterFM)( map_shmem );
   3236    while (VG_(nextIterFM)( map_shmem, NULL, &secmapW )) {
   3237       UWord   j;
   3238       SecMap* sm = (SecMap*)secmapW;
   3239       tl_assert(sm->magic == SecMap_MAGIC);
   3240       /* Deal with the LineZs */
   3241       for (i = 0; i < N_SECMAP_ZLINES; i++) {
   3242          LineZ* lineZ = &sm->linesZ[i];
   3243          if (lineZ->dict[0] != SVal_INVALID) {
   3244             for (j = 0; j < 4; j++)
   3245                remap_VtsIDs_in_SVal(vts_tab, new_tab, &lineZ->dict[j]);
   3246          } else {
   3247             LineF* lineF = SVal2Ptr (lineZ->dict[1]);
   3248             for (j = 0; j < N_LINE_ARANGE; j++)
   3249                remap_VtsIDs_in_SVal(vts_tab, new_tab, &lineF->w64s[j]);
   3250          }
   3251       }
   3252    }
   3253    VG_(doneIterFM)( map_shmem );
   3254 
   3255    /* Do the mappings for (b) above: visit our collection of struct
   3256       _Thrs. */
   3257    Thread* hgthread = get_admin_threads();
   3258    tl_assert(hgthread);
   3259    while (hgthread) {
   3260       Thr* hbthr = hgthread->hbthr;
   3261       tl_assert(hbthr);
   3262       /* Threads that are listed in the prunable set have their viR
   3263          and viW set to VtsID_INVALID, so we can't mess with them. */
   3264       if (hbthr->llexit_done && hbthr->joinedwith_done) {
   3265          tl_assert(hbthr->viR == VtsID_INVALID);
   3266          tl_assert(hbthr->viW == VtsID_INVALID);
   3267          hgthread = hgthread->admin;
   3268          continue;
   3269       }
   3270       remap_VtsID( vts_tab, new_tab, &hbthr->viR );
   3271       remap_VtsID( vts_tab, new_tab, &hbthr->viW );
   3272       hgthread = hgthread->admin;
   3273    }
   3274 
   3275    /* Do the mappings for (c) above: visit the struct _SOs. */
   3276    SO* so = admin_SO;
   3277    while (so) {
   3278       if (so->viR != VtsID_INVALID)
   3279          remap_VtsID( vts_tab, new_tab, &so->viR );
   3280       if (so->viW != VtsID_INVALID)
   3281          remap_VtsID( vts_tab, new_tab, &so->viW );
   3282       so = so->admin_next;
   3283    }
   3284 
   3285    /* So, we're nearly done (with this incredibly complex operation).
   3286       Check the refcounts for the old VtsIDs all fell to zero, as
   3287       expected.  Any failure is serious. */
   3288    for (i = 0; i < nTab; i++) {
   3289       VtsTE* te = VG_(indexXA)( vts_tab, i );
   3290       tl_assert(te->vts == NULL);
   3291       /* This is the assert proper.  Note we're also asserting
   3292          zeroness for old entries which are unmapped.  That's OK. */
   3293       tl_assert(te->rc == 0);
   3294    }
   3295 
   3296    /* Install the new table and set. */
   3297    VG_(deleteFM)(vts_set, NULL/*kFin*/, NULL/*vFin*/);
   3298    vts_set = new_set;
   3299    VG_(deleteXA)( vts_tab );
   3300    vts_tab = new_tab;
   3301 
   3302    /* The freelist of vts_tab entries is empty now, because we've
   3303       compacted all of the live entries at the low end of the
   3304       table. */
   3305    vts_tab_freelist = VtsID_INVALID;
   3306 
   3307    /* Sanity check vts_set and vts_tab. */
   3308 
   3309    /* Because all the live entries got slid down to the bottom of vts_tab: */
   3310    tl_assert( VG_(sizeXA)( vts_tab ) == VG_(sizeFM)( vts_set ));
   3311 
   3312    /* Assert that the vts_tab and vts_set entries point at each other
   3313       in the required way */
   3314    UWord wordK = 0, wordV = 0;
   3315    VG_(initIterFM)( vts_set );
   3316    while (VG_(nextIterFM)( vts_set, &wordK, &wordV )) {
   3317       tl_assert(wordK != 0);
   3318       tl_assert(wordV == 0);
   3319       VTS* vts = (VTS*)wordK;
   3320       tl_assert(vts->id != VtsID_INVALID);
   3321       VtsTE* te = VG_(indexXA)( vts_tab, vts->id );
   3322       tl_assert(te->vts == vts);
   3323    }
   3324    VG_(doneIterFM)( vts_set );
   3325 
   3326    /* Also iterate over the table, and check each entry is
   3327       plausible. */
   3328    nTab = VG_(sizeXA)( vts_tab );
   3329    for (i = 0; i < nTab; i++) {
   3330       VtsTE* te = VG_(indexXA)( vts_tab, i );
   3331       tl_assert(te->vts);
   3332       tl_assert(te->vts->id == i);
   3333       tl_assert(te->rc > 0); /* 'cos we just GC'd */
   3334       tl_assert(te->u.freelink == VtsID_INVALID); /* in use */
   3335       /* value of te->u.remap  not relevant */
   3336    }
   3337 
   3338    /* And we're done.  Bwahahaha. Ha. Ha. Ha. */
   3339    stats__vts_pruning++;
   3340    if (VG_(clo_stats)) {
   3341       tl_assert(nTab > 0);
   3342       VG_(message)(
   3343          Vg_DebugMsg,
   3344          "libhb: VTS PR: #%lu  before %lu (avg sz %lu)  "
   3345             "after %lu (avg sz %lu)\n",
   3346          stats__vts_pruning,
   3347          nBeforePruning, nSTSsBefore / (nBeforePruning ? nBeforePruning : 1),
   3348          nAfterPruning, nSTSsAfter / (nAfterPruning ? nAfterPruning : 1)
   3349       );
   3350    }
   3351    /* ---------- END VTS PRUNING ---------- */
   3352 }
   3353 
   3354 
   3355 /////////////////////////////////////////////////////////
   3356 //                                                     //
   3357 // Vts IDs                                             //
   3358 //                                                     //
   3359 /////////////////////////////////////////////////////////
   3360 
   3361 //////////////////////////
   3362 /* A temporary, max-sized VTS which is used as a temporary (the first
   3363    argument) in VTS__singleton, VTS__tick and VTS__join operations. */
   3364 static VTS* temp_max_sized_VTS = NULL;
   3365 
   3366 //////////////////////////
   3367 static ULong stats__cmpLEQ_queries = 0;
   3368 static ULong stats__cmpLEQ_misses  = 0;
   3369 static ULong stats__join2_queries  = 0;
   3370 static ULong stats__join2_misses   = 0;
   3371 
   3372 static inline UInt ROL32 ( UInt w, Int n ) {
   3373    w = (w << n) | (w >> (32-n));
   3374    return w;
   3375 }
   3376 static inline UInt hash_VtsIDs ( VtsID vi1, VtsID vi2, UInt nTab ) {
   3377    UInt hash = ROL32(vi1,19) ^ ROL32(vi2,13);
   3378    return hash % nTab;
   3379 }
   3380 
   3381 #define N_CMPLEQ_CACHE 1023
   3382 static
   3383    struct { VtsID vi1; VtsID vi2; Bool leq; }
   3384    cmpLEQ_cache[N_CMPLEQ_CACHE];
   3385 
   3386 #define N_JOIN2_CACHE 1023
   3387 static
   3388    struct { VtsID vi1; VtsID vi2; VtsID res; }
   3389    join2_cache[N_JOIN2_CACHE];
   3390 
   3391 static void VtsID__invalidate_caches ( void ) {
   3392    Int i;
   3393    for (i = 0; i < N_CMPLEQ_CACHE; i++) {
   3394       cmpLEQ_cache[i].vi1 = VtsID_INVALID;
   3395       cmpLEQ_cache[i].vi2 = VtsID_INVALID;
   3396       cmpLEQ_cache[i].leq = False;
   3397    }
   3398    for (i = 0; i < N_JOIN2_CACHE; i++) {
   3399      join2_cache[i].vi1 = VtsID_INVALID;
   3400      join2_cache[i].vi2 = VtsID_INVALID;
   3401      join2_cache[i].res = VtsID_INVALID;
   3402    }
   3403 }
   3404 //////////////////////////
   3405 
   3406 //static Bool VtsID__is_valid ( VtsID vi ) {
   3407 //   VtsTE* ve;
   3408 //   if (vi >= (VtsID)VG_(sizeXA)( vts_tab ))
   3409 //      return False;
   3410 //   ve = VG_(indexXA)( vts_tab, vi );
   3411 //   if (!ve->vts)
   3412 //      return False;
   3413 //   tl_assert(ve->vts->id == vi);
   3414 //   return True;
   3415 //}
   3416 
   3417 static VTS* VtsID__to_VTS ( VtsID vi ) {
   3418    VtsTE* te = VG_(indexXA)( vts_tab, vi );
   3419    tl_assert(te->vts);
   3420    return te->vts;
   3421 }
   3422 
   3423 static void VtsID__pp ( VtsID vi ) {
   3424    VTS* vts = VtsID__to_VTS(vi);
   3425    VTS__show( vts );
   3426 }
   3427 
   3428 /* compute partial ordering relation of vi1 and vi2. */
   3429 __attribute__((noinline))
   3430 static Bool VtsID__cmpLEQ_WRK ( VtsID vi1, VtsID vi2 ) {
   3431    UInt hash;
   3432    Bool leq;
   3433    VTS  *v1, *v2;
   3434    //if (vi1 == vi2) return True;
   3435    tl_assert(vi1 != vi2);
   3436    ////++
   3437    stats__cmpLEQ_queries++;
   3438    hash = hash_VtsIDs(vi1, vi2, N_CMPLEQ_CACHE);
   3439    if (cmpLEQ_cache[hash].vi1 == vi1
   3440        && cmpLEQ_cache[hash].vi2 == vi2)
   3441       return cmpLEQ_cache[hash].leq;
   3442    stats__cmpLEQ_misses++;
   3443    ////--
   3444    v1  = VtsID__to_VTS(vi1);
   3445    v2  = VtsID__to_VTS(vi2);
   3446    leq = VTS__cmpLEQ( v1, v2 ) == 0;
   3447    ////++
   3448    cmpLEQ_cache[hash].vi1 = vi1;
   3449    cmpLEQ_cache[hash].vi2 = vi2;
   3450    cmpLEQ_cache[hash].leq = leq;
   3451    ////--
   3452    return leq;
   3453 }
   3454 static inline Bool VtsID__cmpLEQ ( VtsID vi1, VtsID vi2 ) {
   3455    return LIKELY(vi1 == vi2)  ? True  : VtsID__cmpLEQ_WRK(vi1, vi2);
   3456 }
   3457 
   3458 /* compute binary join */
   3459 __attribute__((noinline))
   3460 static VtsID VtsID__join2_WRK ( VtsID vi1, VtsID vi2 ) {
   3461    UInt  hash;
   3462    VtsID res;
   3463    VTS   *vts1, *vts2;
   3464    //if (vi1 == vi2) return vi1;
   3465    tl_assert(vi1 != vi2);
   3466    ////++
   3467    stats__join2_queries++;
   3468    hash = hash_VtsIDs(vi1, vi2, N_JOIN2_CACHE);
   3469    if (join2_cache[hash].vi1 == vi1
   3470        && join2_cache[hash].vi2 == vi2)
   3471       return join2_cache[hash].res;
   3472    stats__join2_misses++;
   3473    ////--
   3474    vts1 = VtsID__to_VTS(vi1);
   3475    vts2 = VtsID__to_VTS(vi2);
   3476    temp_max_sized_VTS->usedTS = 0;
   3477    VTS__join(temp_max_sized_VTS, vts1,vts2);
   3478    res = vts_tab__find__or__clone_and_add(temp_max_sized_VTS);
   3479    ////++
   3480    join2_cache[hash].vi1 = vi1;
   3481    join2_cache[hash].vi2 = vi2;
   3482    join2_cache[hash].res = res;
   3483    ////--
   3484    return res;
   3485 }
   3486 static inline VtsID VtsID__join2 ( VtsID vi1, VtsID vi2 ) {
   3487    return LIKELY(vi1 == vi2)  ? vi1  : VtsID__join2_WRK(vi1, vi2);
   3488 }
   3489 
   3490 /* create a singleton VTS, namely [thr:1] */
   3491 static VtsID VtsID__mk_Singleton ( Thr* thr, ULong tym ) {
   3492    temp_max_sized_VTS->usedTS = 0;
   3493    VTS__singleton(temp_max_sized_VTS, thr,tym);
   3494    return vts_tab__find__or__clone_and_add(temp_max_sized_VTS);
   3495 }
   3496 
   3497 /* tick operation, creates value 1 if specified index is absent */
   3498 static VtsID VtsID__tick ( VtsID vi, Thr* idx ) {
   3499    VTS* vts = VtsID__to_VTS(vi);
   3500    temp_max_sized_VTS->usedTS = 0;
   3501    VTS__tick(temp_max_sized_VTS, idx,vts);
   3502    return vts_tab__find__or__clone_and_add(temp_max_sized_VTS);
   3503 }
   3504 
   3505 /* index into a VTS (only for assertions) */
   3506 static ULong VtsID__indexAt ( VtsID vi, Thr* idx ) {
   3507    VTS* vts = VtsID__to_VTS(vi);
   3508    return VTS__indexAt_SLOW( vts, idx );
   3509 }
   3510 
   3511 /* Assuming that !cmpLEQ(vi1, vi2), find the index of the first (or
   3512    any, really) element in vi1 which is pointwise greater-than the
   3513    corresponding element in vi2.  If no such element exists, return
   3514    NULL.  This needs to be fairly quick since it is called every time
   3515    a race is detected. */
   3516 static Thr* VtsID__findFirst_notLEQ ( VtsID vi1, VtsID vi2 )
   3517 {
   3518    VTS  *vts1, *vts2;
   3519    Thr*  diffthr;
   3520    ThrID diffthrid;
   3521    tl_assert(vi1 != vi2);
   3522    vts1 = VtsID__to_VTS(vi1);
   3523    vts2 = VtsID__to_VTS(vi2);
   3524    tl_assert(vts1 != vts2);
   3525    diffthrid = VTS__cmpLEQ(vts1, vts2);
   3526    diffthr = Thr__from_ThrID(diffthrid);
   3527    tl_assert(diffthr); /* else they are LEQ ! */
   3528    return diffthr;
   3529 }
   3530 
   3531 
   3532 /////////////////////////////////////////////////////////
   3533 //                                                     //
   3534 // Filters                                             //
   3535 //                                                     //
   3536 /////////////////////////////////////////////////////////
   3537 
   3538 /* Forget everything we know -- clear the filter and let everything
   3539    through.  This needs to be as fast as possible, since it is called
   3540    every time the running thread changes, and every time a thread's
   3541    vector clocks change, which can be quite frequent.  The obvious
   3542    fast way to do this is simply to stuff in tags which we know are
   3543    not going to match anything, since they're not aligned to the start
   3544    of a line. */
   3545 static void Filter__clear ( Filter* fi, const HChar* who )
   3546 {
   3547    UWord i;
   3548    if (0) VG_(printf)("  Filter__clear(%p, %s)\n", fi, who);
   3549    for (i = 0; i < FI_NUM_LINES; i += 8) {
   3550       fi->tags[i+0] = 1; /* impossible value -- cannot match */
   3551       fi->tags[i+1] = 1;
   3552       fi->tags[i+2] = 1;
   3553       fi->tags[i+3] = 1;
   3554       fi->tags[i+4] = 1;
   3555       fi->tags[i+5] = 1;
   3556       fi->tags[i+6] = 1;
   3557       fi->tags[i+7] = 1;
   3558    }
   3559    tl_assert(i == FI_NUM_LINES);
   3560 }
   3561 
   3562 /* Clearing an arbitrary range in the filter.  Unfortunately
   3563    we have to do this due to core-supplied new/die-mem events. */
   3564 
   3565 static void Filter__clear_1byte ( Filter* fi, Addr a )
   3566 {
   3567    Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3568    UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3569    FiLine* line   = &fi->lines[lineno];
   3570    UWord   loff   = (a - atag) / 8;
   3571    UShort  mask   = 0x3 << (2 * (a & 7));
   3572    /* mask is C000, 3000, 0C00, 0300, 00C0, 0030, 000C or 0003 */
   3573    if (LIKELY( fi->tags[lineno] == atag )) {
   3574       /* hit.  clear the bits. */
   3575       UShort  u16  = line->u16s[loff];
   3576       line->u16s[loff] = u16 & ~mask; /* clear them */
   3577    } else {
   3578       /* miss.  The filter doesn't hold this address, so ignore. */
   3579    }
   3580 }
   3581 
   3582 static void Filter__clear_8bytes_aligned ( Filter* fi, Addr a )
   3583 {
   3584    Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3585    UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3586    FiLine* line   = &fi->lines[lineno];
   3587    UWord   loff   = (a - atag) / 8;
   3588    if (LIKELY( fi->tags[lineno] == atag )) {
   3589       line->u16s[loff] = 0;
   3590    } else {
   3591     /* miss.  The filter doesn't hold this address, so ignore. */
   3592    }
   3593 }
   3594 
   3595 /* Only used to verify the fast Filter__clear_range */
   3596 __attribute__((unused))
   3597 static void Filter__clear_range_SLOW ( Filter* fi, Addr a, UWord len )
   3598 {
   3599    tl_assert (CHECK_ZSM);
   3600 
   3601    /* slowly do part preceding 8-alignment */
   3602    while (UNLIKELY(!VG_IS_8_ALIGNED(a)) && LIKELY(len > 0)) {
   3603       Filter__clear_1byte( fi, a );
   3604       a++;
   3605       len--;
   3606    }
   3607    /* vector loop */
   3608    while (len >= 8) {
   3609       Filter__clear_8bytes_aligned( fi, a );
   3610       a += 8;
   3611       len -= 8;
   3612    }
   3613    /* slowly do tail */
   3614    while (UNLIKELY(len > 0)) {
   3615       Filter__clear_1byte( fi, a );
   3616       a++;
   3617       len--;
   3618    }
   3619 }
   3620 
   3621 static void Filter__clear_range ( Filter* fi, Addr a, UWord len )
   3622 {
   3623 #  if CHECK_ZSM > 0
   3624    /* We check the below more complex algorithm with the simple one.
   3625       This check is very expensive : we do first the slow way on a
   3626       copy of the data, then do it the fast way. On RETURN, we check
   3627       the two values are equal. */
   3628    Filter fi_check = *fi;
   3629    Filter__clear_range_SLOW(&fi_check, a, len);
   3630 #  define RETURN goto check_and_return
   3631 #  else
   3632 #  define RETURN return
   3633 #  endif
   3634 
   3635    Addr    begtag = FI_GET_TAG(a);       /* tag of range begin */
   3636 
   3637    Addr    end = a + len - 1;
   3638    Addr    endtag = FI_GET_TAG(end); /* tag of range end. */
   3639 
   3640    UWord rlen = len; /* remaining length to clear */
   3641 
   3642    Addr    c = a; /* Current position we are clearing. */
   3643    UWord   clineno = FI_GET_LINENO(c); /* Current lineno we are clearing */
   3644    FiLine* cline; /* Current line we are clearing */
   3645    UWord   cloff; /* Current offset in line we are clearing, when clearing
   3646                      partial lines. */
   3647 
   3648    UShort u16;
   3649 
   3650    STATIC_ASSERT (FI_LINE_SZB == 32);
   3651    // Below assumes filter lines are 32 bytes
   3652 
   3653    if (LIKELY(fi->tags[clineno] == begtag)) {
   3654       /* LIKELY for the heavy caller VG_(unknown_SP_update). */
   3655       /* First filter line matches begtag.
   3656          If c is not at the filter line begin, the below will clear
   3657          the filter line bytes starting from c. */
   3658       cline = &fi->lines[clineno];
   3659       cloff = (c - begtag) / 8;
   3660 
   3661       /* First the byte(s) needed to reach 8-alignment */
   3662       if (UNLIKELY(!VG_IS_8_ALIGNED(c))) {
   3663          /* hiB is the nr of bytes (higher addresses) from c to reach
   3664             8-aligment. */
   3665          UWord hiB = 8 - (c & 7);
   3666          /* Compute 2-bit/byte mask representing hiB bytes [c..c+hiB[
   3667             mask is  C000 , F000, FC00, FF00, FFC0, FFF0 or FFFC for the byte
   3668             range    7..7   6..7  5..7  4..7  3..7  2..7    1..7 */
   3669          UShort mask = 0xFFFF << (16 - 2*hiB);
   3670 
   3671          u16  = cline->u16s[cloff];
   3672          if (LIKELY(rlen >= hiB)) {
   3673             cline->u16s[cloff] = u16 & ~mask; /* clear all hiB from c */
   3674             rlen -= hiB;
   3675             c += hiB;
   3676             cloff += 1;
   3677          } else {
   3678             /* Only have the bits for rlen bytes bytes. */
   3679             mask = mask & ~(0xFFFF << (16 - 2*(hiB-rlen)));
   3680             cline->u16s[cloff] = u16 & ~mask; /* clear rlen bytes from c. */
   3681             RETURN;  // We have cleared all what we can.
   3682          }
   3683       }
   3684       /* c is now 8 aligned. Clear by 8 aligned bytes,
   3685          till c is filter-line aligned */
   3686       while (!VG_IS_32_ALIGNED(c) && rlen >= 8) {
   3687          cline->u16s[cloff] = 0;
   3688          c += 8;
   3689          rlen -= 8;
   3690          cloff += 1;
   3691       }
   3692    } else {
   3693       c = begtag + FI_LINE_SZB;
   3694       if (c > end)
   3695          RETURN;   // We have cleared all what we can.
   3696       rlen -= c - a;
   3697    }
   3698    // We have changed c, so re-establish clineno.
   3699    clineno = FI_GET_LINENO(c);
   3700 
   3701    if (rlen >= FI_LINE_SZB) {
   3702       /* Here, c is filter line-aligned. Clear all full lines that
   3703          overlap with the range starting at c, made of a full lines */
   3704       UWord nfull = rlen / FI_LINE_SZB;
   3705       UWord full_len = nfull * FI_LINE_SZB;
   3706       rlen -= full_len;
   3707       if (nfull > FI_NUM_LINES)
   3708          nfull = FI_NUM_LINES; // no need to check several times the same entry.
   3709 
   3710       for (UWord n = 0; n < nfull; n++) {
   3711          if (UNLIKELY(address_in_range(fi->tags[clineno], c, full_len))) {
   3712             cline = &fi->lines[clineno];
   3713             cline->u16s[0] = 0;
   3714             cline->u16s[1] = 0;
   3715             cline->u16s[2] = 0;
   3716             cline->u16s[3] = 0;
   3717             STATIC_ASSERT (4 == sizeof(cline->u16s)/sizeof(cline->u16s[0]));
   3718          }
   3719          clineno++;
   3720          if (UNLIKELY(clineno == FI_NUM_LINES))
   3721             clineno = 0;
   3722       }
   3723 
   3724       c += full_len;
   3725       clineno = FI_GET_LINENO(c);
   3726    }
   3727 
   3728    if (CHECK_ZSM) {
   3729       tl_assert(VG_IS_8_ALIGNED(c));
   3730       tl_assert(clineno == FI_GET_LINENO(c));
   3731    }
   3732 
   3733    /* Do the last filter line, if it was not cleared as a full filter line */
   3734    if (UNLIKELY(rlen > 0) && fi->tags[clineno] == endtag) {
   3735       cline = &fi->lines[clineno];
   3736       cloff = (c - endtag) / 8;
   3737       if (CHECK_ZSM) tl_assert(FI_GET_TAG(c) == endtag);
   3738 
   3739       /* c is 8 aligned. Clear by 8 aligned bytes, till we have less than
   3740          8 bytes. */
   3741       while (rlen >= 8) {
   3742          cline->u16s[cloff] = 0;
   3743          c += 8;
   3744          rlen -= 8;
   3745          cloff += 1;
   3746       }
   3747       /* Then the remaining byte(s) */
   3748       if (rlen > 0) {
   3749          /* nr of bytes from c to reach end. */
   3750          UWord loB = rlen;
   3751          /* Compute mask representing loB bytes [c..c+loB[ :
   3752             mask is 0003, 000F, 003F, 00FF, 03FF, 0FFF or 3FFF */
   3753          UShort mask = 0xFFFF >> (16 - 2*loB);
   3754 
   3755          u16  = cline->u16s[cloff];
   3756          cline->u16s[cloff] = u16 & ~mask; /* clear all loB from c */
   3757       }
   3758    }
   3759 
   3760 #  if CHECK_ZSM > 0
   3761    check_and_return:
   3762    tl_assert (VG_(memcmp)(&fi_check, fi, sizeof(fi_check)) == 0);
   3763 #  endif
   3764 #  undef RETURN
   3765 }
   3766 
   3767 /* ------ Read handlers for the filter. ------ */
   3768 
   3769 static inline Bool Filter__ok_to_skip_crd64 ( Filter* fi, Addr a )
   3770 {
   3771    if (UNLIKELY( !VG_IS_8_ALIGNED(a) ))
   3772       return False;
   3773    {
   3774      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3775      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3776      FiLine* line   = &fi->lines[lineno];
   3777      UWord   loff   = (a - atag) / 8;
   3778      UShort  mask   = 0xAAAA;
   3779      if (LIKELY( fi->tags[lineno] == atag )) {
   3780         /* hit.  check line and update. */
   3781         UShort u16  = line->u16s[loff];
   3782         Bool   ok   = (u16 & mask) == mask; /* all R bits set? */
   3783         line->u16s[loff] = u16 | mask; /* set them */
   3784         return ok;
   3785      } else {
   3786         /* miss.  nuke existing line and re-use it. */
   3787         UWord i;
   3788         fi->tags[lineno] = atag;
   3789         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3790            line->u16s[i] = 0;
   3791         line->u16s[loff] = mask;
   3792         return False;
   3793      }
   3794    }
   3795 }
   3796 
   3797 static inline Bool Filter__ok_to_skip_crd32 ( Filter* fi, Addr a )
   3798 {
   3799    if (UNLIKELY( !VG_IS_4_ALIGNED(a) ))
   3800       return False;
   3801    {
   3802      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3803      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3804      FiLine* line   = &fi->lines[lineno];
   3805      UWord   loff   = (a - atag) / 8;
   3806      UShort  mask   = 0xAA << (2 * (a & 4)); /* 0xAA00 or 0x00AA */
   3807      if (LIKELY( fi->tags[lineno] == atag )) {
   3808         /* hit.  check line and update. */
   3809         UShort  u16  = line->u16s[loff];
   3810         Bool    ok   = (u16 & mask) == mask; /* 4 x R bits set? */
   3811         line->u16s[loff] = u16 | mask; /* set them */
   3812         return ok;
   3813      } else {
   3814         /* miss.  nuke existing line and re-use it. */
   3815         UWord   i;
   3816         fi->tags[lineno] = atag;
   3817         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3818            line->u16s[i] = 0;
   3819         line->u16s[loff] = mask;
   3820         return False;
   3821      }
   3822    }
   3823 }
   3824 
   3825 static inline Bool Filter__ok_to_skip_crd16 ( Filter* fi, Addr a )
   3826 {
   3827    if (UNLIKELY( !VG_IS_2_ALIGNED(a) ))
   3828       return False;
   3829    {
   3830      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3831      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3832      FiLine* line   = &fi->lines[lineno];
   3833      UWord   loff   = (a - atag) / 8;
   3834      UShort  mask   = 0xA << (2 * (a & 6));
   3835      /* mask is A000, 0A00, 00A0 or 000A */
   3836      if (LIKELY( fi->tags[lineno] == atag )) {
   3837         /* hit.  check line and update. */
   3838         UShort  u16  = line->u16s[loff];
   3839         Bool    ok   = (u16 & mask) == mask; /* 2 x R bits set? */
   3840         line->u16s[loff] = u16 | mask; /* set them */
   3841         return ok;
   3842      } else {
   3843         /* miss.  nuke existing line and re-use it. */
   3844         UWord   i;
   3845         fi->tags[lineno] = atag;
   3846         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3847            line->u16s[i] = 0;
   3848         line->u16s[loff] = mask;
   3849         return False;
   3850      }
   3851    }
   3852 }
   3853 
   3854 static inline Bool Filter__ok_to_skip_crd08 ( Filter* fi, Addr a )
   3855 {
   3856    {
   3857      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3858      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3859      FiLine* line   = &fi->lines[lineno];
   3860      UWord   loff   = (a - atag) / 8;
   3861      UShort  mask   = 0x2 << (2 * (a & 7));
   3862      /* mask is 8000, 2000, 0800, 0200, 0080, 0020, 0008 or 0002 */
   3863      if (LIKELY( fi->tags[lineno] == atag )) {
   3864         /* hit.  check line and update. */
   3865         UShort  u16  = line->u16s[loff];
   3866         Bool    ok   = (u16 & mask) == mask; /* 1 x R bits set? */
   3867         line->u16s[loff] = u16 | mask; /* set them */
   3868         return ok;
   3869      } else {
   3870         /* miss.  nuke existing line and re-use it. */
   3871         UWord   i;
   3872         fi->tags[lineno] = atag;
   3873         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3874            line->u16s[i] = 0;
   3875         line->u16s[loff] = mask;
   3876         return False;
   3877      }
   3878    }
   3879 }
   3880 
   3881 
   3882 /* ------ Write handlers for the filter. ------ */
   3883 
   3884 static inline Bool Filter__ok_to_skip_cwr64 ( Filter* fi, Addr a )
   3885 {
   3886    if (UNLIKELY( !VG_IS_8_ALIGNED(a) ))
   3887       return False;
   3888    {
   3889      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3890      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3891      FiLine* line   = &fi->lines[lineno];
   3892      UWord   loff   = (a - atag) / 8;
   3893      UShort  mask   = 0xFFFF;
   3894      if (LIKELY( fi->tags[lineno] == atag )) {
   3895         /* hit.  check line and update. */
   3896         UShort u16  = line->u16s[loff];
   3897         Bool   ok   = (u16 & mask) == mask; /* all R & W bits set? */
   3898         line->u16s[loff] = u16 | mask; /* set them */
   3899         return ok;
   3900      } else {
   3901         /* miss.  nuke existing line and re-use it. */
   3902         UWord i;
   3903         fi->tags[lineno] = atag;
   3904         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3905            line->u16s[i] = 0;
   3906         line->u16s[loff] = mask;
   3907         return False;
   3908      }
   3909    }
   3910 }
   3911 
   3912 static inline Bool Filter__ok_to_skip_cwr32 ( Filter* fi, Addr a )
   3913 {
   3914    if (UNLIKELY( !VG_IS_4_ALIGNED(a) ))
   3915       return False;
   3916    {
   3917      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3918      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3919      FiLine* line   = &fi->lines[lineno];
   3920      UWord   loff   = (a - atag) / 8;
   3921      UShort  mask   = 0xFF << (2 * (a & 4)); /* 0xFF00 or 0x00FF */
   3922      if (LIKELY( fi->tags[lineno] == atag )) {
   3923         /* hit.  check line and update. */
   3924         UShort  u16  = line->u16s[loff];
   3925         Bool    ok   = (u16 & mask) == mask; /* 4 x R & W bits set? */
   3926         line->u16s[loff] = u16 | mask; /* set them */
   3927         return ok;
   3928      } else {
   3929         /* miss.  nuke existing line and re-use it. */
   3930         UWord   i;
   3931         fi->tags[lineno] = atag;
   3932         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3933            line->u16s[i] = 0;
   3934         line->u16s[loff] = mask;
   3935         return False;
   3936      }
   3937    }
   3938 }
   3939 
   3940 static inline Bool Filter__ok_to_skip_cwr16 ( Filter* fi, Addr a )
   3941 {
   3942    if (UNLIKELY( !VG_IS_2_ALIGNED(a) ))
   3943       return False;
   3944    {
   3945      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3946      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3947      FiLine* line   = &fi->lines[lineno];
   3948      UWord   loff   = (a - atag) / 8;
   3949      UShort  mask   = 0xF << (2 * (a & 6));
   3950      /* mask is F000, 0F00, 00F0 or 000F */
   3951      if (LIKELY( fi->tags[lineno] == atag )) {
   3952         /* hit.  check line and update. */
   3953         UShort  u16  = line->u16s[loff];
   3954         Bool    ok   = (u16 & mask) == mask; /* 2 x R & W bits set? */
   3955         line->u16s[loff] = u16 | mask; /* set them */
   3956         return ok;
   3957      } else {
   3958         /* miss.  nuke existing line and re-use it. */
   3959         UWord   i;
   3960         fi->tags[lineno] = atag;
   3961         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3962            line->u16s[i] = 0;
   3963         line->u16s[loff] = mask;
   3964         return False;
   3965      }
   3966    }
   3967 }
   3968 
   3969 static inline Bool Filter__ok_to_skip_cwr08 ( Filter* fi, Addr a )
   3970 {
   3971    {
   3972      Addr    atag   = FI_GET_TAG(a);     /* tag of 'a' */
   3973      UWord   lineno = FI_GET_LINENO(a);  /* lineno for 'a' */
   3974      FiLine* line   = &fi->lines[lineno];
   3975      UWord   loff   = (a - atag) / 8;
   3976      UShort  mask   = 0x3 << (2 * (a & 7));
   3977      /* mask is C000, 3000, 0C00, 0300, 00C0, 0030, 000C or 0003 */
   3978      if (LIKELY( fi->tags[lineno] == atag )) {
   3979         /* hit.  check line and update. */
   3980         UShort  u16  = line->u16s[loff];
   3981         Bool    ok   = (u16 & mask) == mask; /* 1 x R bits set? */
   3982         line->u16s[loff] = u16 | mask; /* set them */
   3983         return ok;
   3984      } else {
   3985         /* miss.  nuke existing line and re-use it. */
   3986         UWord   i;
   3987         fi->tags[lineno] = atag;
   3988         for (i = 0; i < FI_LINE_SZB / 8; i++)
   3989            line->u16s[i] = 0;
   3990         line->u16s[loff] = mask;
   3991         return False;
   3992      }
   3993    }
   3994 }
   3995 
   3996 
   3997 /////////////////////////////////////////////////////////
   3998 //                                                     //
   3999 // Threads                                             //
   4000 //                                                     //
   4001 /////////////////////////////////////////////////////////
   4002 
   4003 /* Maps ThrID values to their Thr*s (which contain ThrID values that
   4004    should point back to the relevant slot in the array.  Lowest
   4005    numbered slot (0) is for thrid = 1024, (1) is for 1025, etc. */
   4006 static XArray* /* of Thr* */ thrid_to_thr_map = NULL;
   4007 
   4008 /* And a counter to dole out ThrID values.  For rationale/background,
   4009    see comments on definition of ScalarTS (far) above. */
   4010 static ThrID thrid_counter = 1024; /* runs up to ThrID_MAX_VALID */
   4011 
   4012 static ThrID Thr__to_ThrID ( Thr* thr ) {
   4013    return thr->thrid;
   4014 }
   4015 static Thr* Thr__from_ThrID ( UInt thrid ) {
   4016    Thr* thr = *(Thr**)VG_(indexXA)( thrid_to_thr_map, thrid - 1024 );
   4017    tl_assert(thr->thrid == thrid);
   4018    return thr;
   4019 }
   4020 
   4021 static Thr* Thr__new ( void )
   4022 {
   4023    Thr* thr = HG_(zalloc)( "libhb.Thr__new.1", sizeof(Thr) );
   4024    thr->viR = VtsID_INVALID;
   4025    thr->viW = VtsID_INVALID;
   4026    thr->llexit_done = False;
   4027    thr->joinedwith_done = False;
   4028    thr->filter = HG_(zalloc)( "libhb.Thr__new.2", sizeof(Filter) );
   4029    if (HG_(clo_history_level) == 1)
   4030       thr->local_Kws_n_stacks
   4031          = VG_(newXA)( HG_(zalloc),
   4032                        "libhb.Thr__new.3 (local_Kws_and_stacks)",
   4033                        HG_(free), sizeof(ULong_n_EC) );
   4034 
   4035    /* Add this Thr* <-> ThrID binding to the mapping, and
   4036       cross-check */
   4037    if (!thrid_to_thr_map) {
   4038       thrid_to_thr_map = VG_(newXA)( HG_(zalloc), "libhb.Thr__new.4",
   4039                                      HG_(free), sizeof(Thr*) );
   4040    }
   4041 
   4042    if (thrid_counter >= ThrID_MAX_VALID) {
   4043       /* We're hosed.  We have to stop. */
   4044       scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ );
   4045    }
   4046 
   4047    thr->thrid = thrid_counter++;
   4048    Word ix = VG_(addToXA)( thrid_to_thr_map, &thr );
   4049    tl_assert(ix + 1024 == thr->thrid);
   4050 
   4051    return thr;
   4052 }
   4053 
   4054 static void note_local_Kw_n_stack_for ( Thr* thr )
   4055 {
   4056    Word       nPresent;
   4057    ULong_n_EC pair;
   4058    tl_assert(thr);
   4059 
   4060    // We only collect this info at history level 1 (approx)
   4061    if (HG_(clo_history_level) != 1)
   4062       return;
   4063 
   4064    /* This is the scalar Kw for thr. */
   4065    pair.ull = VtsID__indexAt( thr->viW, thr );
   4066    pair.ec  = main_get_EC( thr );
   4067    tl_assert(pair.ec);
   4068    tl_assert(thr->local_Kws_n_stacks);
   4069 
   4070    /* check that we're not adding duplicates */
   4071    nPresent = VG_(sizeXA)( thr->local_Kws_n_stacks );
   4072 
   4073    /* Throw away old stacks, if necessary.  We can't accumulate stuff
   4074       indefinitely. */
   4075    if (nPresent >= N_KWs_N_STACKs_PER_THREAD) {
   4076       VG_(dropHeadXA)( thr->local_Kws_n_stacks, nPresent / 2 );
   4077       nPresent = VG_(sizeXA)( thr->local_Kws_n_stacks );
   4078       if (0)
   4079          VG_(printf)("LOCAL Kw: thr %p,  Kw %llu,  ec %p (!!! gc !!!)\n",
   4080                      thr, pair.ull, pair.ec );
   4081    }
   4082 
   4083    if (nPresent > 0) {
   4084       ULong_n_EC* prevPair
   4085          = (ULong_n_EC*)VG_(indexXA)( thr->local_Kws_n_stacks, nPresent-1 );
   4086       tl_assert( prevPair->ull <= pair.ull );
   4087    }
   4088 
   4089    if (nPresent == 0)
   4090       pair.ec = NULL;
   4091 
   4092    VG_(addToXA)( thr->local_Kws_n_stacks, &pair );
   4093 
   4094    if (0)
   4095       VG_(printf)("LOCAL Kw: thr %p,  Kw %llu,  ec %p\n",
   4096                   thr, pair.ull, pair.ec );
   4097    if (0)
   4098       VG_(pp_ExeContext)(pair.ec);
   4099 }
   4100 
   4101 static Int cmp__ULong_n_EC__by_ULong ( const ULong_n_EC* pair1,
   4102                                        const ULong_n_EC* pair2 )
   4103 {
   4104    if (pair1->ull < pair2->ull) return -1;
   4105    if (pair1->ull > pair2->ull) return 1;
   4106    return 0;
   4107 }
   4108 
   4109 
   4110 /////////////////////////////////////////////////////////
   4111 //                                                     //
   4112 // Shadow Values                                       //
   4113 //                                                     //
   4114 /////////////////////////////////////////////////////////
   4115 
   4116 // type SVal, SVal_INVALID and SVal_NOACCESS are defined by
   4117 // hb_zsm.h.  We have to do everything else here.
   4118 
   4119 /* SVal is 64 bit unsigned int.
   4120 
   4121       <---------30--------->    <---------30--------->
   4122    00 X-----Rmin-VtsID-----X 00 X-----Wmin-VtsID-----X   C(Rmin,Wmin)
   4123    10 X--------------------X XX X--------------------X   A: SVal_NOACCESS
   4124    11 0--------------------0 00 0--------------------0   A: SVal_INVALID
   4125 
   4126 */
   4127 #define SVAL_TAGMASK (3ULL << 62)
   4128 
   4129 static inline Bool SVal__isC ( SVal s ) {
   4130    return (0ULL << 62) == (s & SVAL_TAGMASK);
   4131 }
   4132 static inline SVal SVal__mkC ( VtsID rmini, VtsID wmini ) {
   4133    //tl_assert(VtsID__is_valid(rmini));
   4134    //tl_assert(VtsID__is_valid(wmini));
   4135    return (((ULong)rmini) << 32) | ((ULong)wmini);
   4136 }
   4137 static inline VtsID SVal__unC_Rmin ( SVal s ) {
   4138    tl_assert(SVal__isC(s));
   4139    return (VtsID)(s >> 32);
   4140 }
   4141 static inline VtsID SVal__unC_Wmin ( SVal s ) {
   4142    tl_assert(SVal__isC(s));
   4143    return (VtsID)(s & 0xFFFFFFFFULL);
   4144 }
   4145 
   4146 static inline Bool SVal__isA ( SVal s ) {
   4147    return (2ULL << 62) == (s & SVAL_TAGMASK);
   4148 }
   4149 __attribute__((unused))
   4150 static inline SVal SVal__mkA ( void ) {
   4151    return 2ULL << 62;
   4152 }
   4153 
   4154 /* Direct callback from lib_zsm. */
   4155 static inline void SVal__rcinc ( SVal s ) {
   4156    if (SVal__isC(s)) {
   4157       VtsID__rcinc( SVal__unC_Rmin(s) );
   4158       VtsID__rcinc( SVal__unC_Wmin(s) );
   4159    }
   4160 }
   4161 
   4162 /* Direct callback from lib_zsm. */
   4163 static inline void SVal__rcdec ( SVal s ) {
   4164    if (SVal__isC(s)) {
   4165       VtsID__rcdec( SVal__unC_Rmin(s) );
   4166       VtsID__rcdec( SVal__unC_Wmin(s) );
   4167    }
   4168 }
   4169 
   4170 static inline void *SVal2Ptr (SVal s)
   4171 {
   4172    return (void*)(UWord)s;
   4173 }
   4174 
   4175 static inline SVal Ptr2SVal (void* ptr)
   4176 {
   4177    return (SVal)(UWord)ptr;
   4178 }
   4179 
   4180 
   4181 
   4182 /////////////////////////////////////////////////////////
   4183 //                                                     //
   4184 // Change-event map2                                   //
   4185 //                                                     //
   4186 /////////////////////////////////////////////////////////
   4187 
   4188 /* This is in two parts:
   4189 
   4190    1. A hash table of RCECs.  This is a set of reference-counted stack
   4191       traces.  When the reference count of a stack trace becomes zero,
   4192       it is removed from the set and freed up.  The intent is to have
   4193       a set of stack traces which can be referred to from (2), but to
   4194       only represent each one once.  The set is indexed/searched by
   4195       ordering on the stack trace vectors.
   4196 
   4197    2. A Hash table of OldRefs.  These store information about each old
   4198       ref that we need to record.  Hash table key is the address of the
   4199       location for which the information is recorded.  For LRU
   4200       purposes, each OldRef in the hash table is also on a doubly
   4201       linked list maintaining the order in which the OldRef were most
   4202       recently accessed.
   4203       Each OldRef also maintains the stamp at which it was last accessed.
   4204       With these stamps, we can quickly check which of 2 OldRef is the
   4205       'newest', without having to scan the full list of LRU OldRef.
   4206 
   4207       The important part of an OldRef is, however, its acc component.
   4208       This binds a TSW triple (thread, size, R/W) to an RCEC.
   4209 
   4210       We allocate a maximum of VG_(clo_conflict_cache_size) OldRef.
   4211       Then we do exact LRU discarding.  For each discarded OldRef we must
   4212       of course decrement the reference count on the RCEC it
   4213       refers to, in order that entries from (1) eventually get
   4214       discarded too.
   4215 */
   4216 
   4217 static UWord stats__evm__lookup_found = 0;
   4218 static UWord stats__evm__lookup_notfound = 0;
   4219 
   4220 static UWord stats__ctxt_eq_tsw_eq_rcec = 0;
   4221 static UWord stats__ctxt_eq_tsw_neq_rcec = 0;
   4222 static UWord stats__ctxt_neq_tsw_neq_rcec = 0;
   4223 static UWord stats__ctxt_rcdec_calls = 0;
   4224 static UWord stats__ctxt_rcec_gc_discards = 0;
   4225 
   4226 static UWord stats__ctxt_tab_curr = 0;
   4227 static UWord stats__ctxt_tab_max  = 0;
   4228 
   4229 static UWord stats__ctxt_tab_qs   = 0;
   4230 static UWord stats__ctxt_tab_cmps = 0;
   4231 
   4232 
   4233 ///////////////////////////////////////////////////////
   4234 //// Part (1): A hash table of RCECs
   4235 ///
   4236 
   4237 #define N_FRAMES 8
   4238 
   4239 // (UInt) `echo "Reference Counted Execution Context" | md5sum`
   4240 #define RCEC_MAGIC 0xab88abb2UL
   4241 
   4242 //#define N_RCEC_TAB 98317 /* prime */
   4243 #define N_RCEC_TAB 196613 /* prime */
   4244 
   4245 typedef
   4246    struct _RCEC {
   4247       UWord magic;  /* sanity check only */
   4248       struct _RCEC* next;
   4249       UWord rc;
   4250       UWord rcX; /* used for crosschecking */
   4251       UWord frames_hash;          /* hash of all the frames */
   4252       UWord frames[N_FRAMES];
   4253    }
   4254    RCEC;
   4255 
   4256 //////////// BEGIN RCEC pool allocator
   4257 static PoolAlloc* rcec_pool_allocator;
   4258 static RCEC* alloc_RCEC ( void ) {
   4259    return VG_(allocEltPA) ( rcec_pool_allocator );
   4260 }
   4261 
   4262 static void free_RCEC ( RCEC* rcec ) {
   4263    tl_assert(rcec->magic == RCEC_MAGIC);
   4264    VG_(freeEltPA)( rcec_pool_allocator, rcec );
   4265 }
   4266 //////////// END RCEC pool allocator
   4267 
   4268 static RCEC** contextTab = NULL; /* hash table of RCEC*s */
   4269 
   4270 /* Count of allocated RCEC having ref count > 0 */
   4271 static UWord RCEC_referenced = 0;
   4272 
   4273 /* Gives an arbitrary total order on RCEC .frames fields */
   4274 static Word RCEC__cmp_by_frames ( RCEC* ec1, RCEC* ec2 ) {
   4275    Word i;
   4276    tl_assert(ec1 && ec1->magic == RCEC_MAGIC);
   4277    tl_assert(ec2 && ec2->magic == RCEC_MAGIC);
   4278    if (ec1->frames_hash < ec2->frames_hash) return -1;
   4279    if (ec1->frames_hash > ec2->frames_hash) return  1;
   4280    for (i = 0; i < N_FRAMES; i++) {
   4281       if (ec1->frames[i] < ec2->frames[i]) return -1;
   4282       if (ec1->frames[i] > ec2->frames[i]) return  1;
   4283    }
   4284    return 0;
   4285 }
   4286 
   4287 
   4288 /* Dec the ref of this RCEC. */
   4289 static void ctxt__rcdec ( RCEC* ec )
   4290 {
   4291    stats__ctxt_rcdec_calls++;
   4292    tl_assert(ec && ec->magic == RCEC_MAGIC);
   4293    tl_assert(ec->rc > 0);
   4294    ec->rc--;
   4295    if (ec->rc == 0)
   4296       RCEC_referenced--;
   4297 }
   4298 
   4299 static void ctxt__rcinc ( RCEC* ec )
   4300 {
   4301    tl_assert(ec && ec->magic == RCEC_MAGIC);
   4302    if (ec->rc == 0)
   4303       RCEC_referenced++;
   4304    ec->rc++;
   4305 }
   4306 
   4307 
   4308 /* Find 'ec' in the RCEC list whose head pointer lives at 'headp' and
   4309    move it one step closer to the front of the list, so as to make
   4310    subsequent searches for it cheaper. */
   4311 static void move_RCEC_one_step_forward ( RCEC** headp, RCEC* ec )
   4312 {
   4313    RCEC *ec0, *ec1, *ec2;
   4314    if (ec == *headp)
   4315       tl_assert(0); /* already at head of list */
   4316    tl_assert(ec != NULL);
   4317    ec0 = *headp;
   4318    ec1 = NULL;
   4319    ec2 = NULL;
   4320    while (True) {
   4321       if (ec0 == NULL || ec0 == ec) break;
   4322       ec2 = ec1;
   4323       ec1 = ec0;
   4324       ec0 = ec0->next;
   4325    }
   4326    tl_assert(ec0 == ec);
   4327    if (ec0 != NULL && ec1 != NULL && ec2 != NULL) {
   4328       RCEC* tmp;
   4329       /* ec0 points to ec, ec1 to its predecessor, and ec2 to ec1's
   4330          predecessor.  Swap ec0 and ec1, that is, move ec0 one step
   4331          closer to the start of the list. */
   4332       tl_assert(ec2->next == ec1);
   4333       tl_assert(ec1->next == ec0);
   4334       tmp = ec0->next;
   4335       ec2->next = ec0;
   4336       ec0->next = ec1;
   4337       ec1->next = tmp;
   4338    }
   4339    else
   4340    if (ec0 != NULL && ec1 != NULL && ec2 == NULL) {
   4341       /* it's second in the list. */
   4342       tl_assert(*headp == ec1);
   4343       tl_assert(ec1->next == ec0);
   4344       ec1->next = ec0->next;
   4345       ec0->next = ec1;
   4346       *headp = ec0;
   4347    }
   4348 }
   4349 
   4350 
   4351 /* Find the given RCEC in the tree, and return a pointer to it.  Or,
   4352    if not present, add the given one to the tree (by making a copy of
   4353    it, so the caller can immediately deallocate the original) and
   4354    return a pointer to the copy.  The caller can safely have 'example'
   4355    on its stack, since we will always return a pointer to a copy of
   4356    it, not to the original.  Note that the inserted node will have .rc
   4357    of zero and so the caller must immediately increment it. */
   4358 __attribute__((noinline))
   4359 static RCEC* ctxt__find_or_add ( RCEC* example )
   4360 {
   4361    UWord hent;
   4362    RCEC* copy;
   4363    tl_assert(example && example->magic == RCEC_MAGIC);
   4364    tl_assert(example->rc == 0);
   4365 
   4366    /* Search the hash table to see if we already have it. */
   4367    stats__ctxt_tab_qs++;
   4368    hent = example->frames_hash % N_RCEC_TAB;
   4369    copy = contextTab[hent];
   4370    while (1) {
   4371       if (!copy) break;
   4372       tl_assert(copy->magic == RCEC_MAGIC);
   4373       stats__ctxt_tab_cmps++;
   4374       if (0 == RCEC__cmp_by_frames(copy, example)) break;
   4375       copy = copy->next;
   4376    }
   4377 
   4378    if (copy) {
   4379       tl_assert(copy != example);
   4380       /* optimisation: if it's not at the head of its list, move 1
   4381          step fwds, to make future searches cheaper */
   4382       if (copy != contextTab[hent]) {
   4383          move_RCEC_one_step_forward( &contextTab[hent], copy );
   4384       }
   4385    } else {
   4386       copy = alloc_RCEC();
   4387       tl_assert(copy != example);
   4388       *copy = *example;
   4389       copy->next = contextTab[hent];
   4390       contextTab[hent] = copy;
   4391       stats__ctxt_tab_curr++;
   4392       if (stats__ctxt_tab_curr > stats__ctxt_tab_max)
   4393          stats__ctxt_tab_max = stats__ctxt_tab_curr;
   4394    }
   4395    return copy;
   4396 }
   4397 
   4398 static inline UWord ROLW ( UWord w, Int n )
   4399 {
   4400    Int bpw = 8 * sizeof(UWord);
   4401    w = (w << n) | (w >> (bpw-n));
   4402    return w;
   4403 }
   4404 
   4405 __attribute__((noinline))
   4406 static RCEC* get_RCEC ( Thr* thr )
   4407 {
   4408    UWord hash, i;
   4409    RCEC  example;
   4410    example.magic = RCEC_MAGIC;
   4411    example.rc = 0;
   4412    example.rcX = 0;
   4413    example.next = NULL;
   4414    main_get_stacktrace( thr, &example.frames[0], N_FRAMES );
   4415    hash = 0;
   4416    for (i = 0; i < N_FRAMES; i++) {
   4417       hash ^= example.frames[i];
   4418       hash = ROLW(hash, 19);
   4419    }
   4420    example.frames_hash = hash;
   4421    return ctxt__find_or_add( &example );
   4422 }
   4423 
   4424 ///////////////////////////////////////////////////////
   4425 //// Part (2):
   4426 ///  A hashtable guest-addr -> OldRef, that refers to (1)
   4427 ///  Note: we use the guest address as key. This means that the entries
   4428 ///  for multiple threads accessing the same address will land in the same
   4429 ///  bucket. It might be nice to have a better distribution of the
   4430 ///  OldRef in the hashtable by using ask key the guestaddress ^ tsw.
   4431 ///  The problem is that when a race is reported on a ga, we need to retrieve
   4432 ///  efficiently the accesses to ga by other threads, only using the ga.
   4433 ///  Measurements on firefox have shown that the chain length is reasonable.
   4434 
   4435 /* Records an access: a thread, a context (size & writeness) and the
   4436    number of held locks. The size (1,2,4,8) is stored as is in szB.
   4437    Note that szB uses more bits than needed to store a size up to 8.
   4438    This allows to use a TSW as a fully initialised UInt e.g. in
   4439    cmp_oldref_tsw. If needed, a more compact representation of szB
   4440    can be done (e.g. use only 4 bits, or use only 2 bits and encode the
   4441    size (1,2,4,8) as 00 = 1, 01 = 2, 10 = 4, 11 = 8. */
   4442 typedef
   4443    struct {
   4444       UInt      thrid  : SCALARTS_N_THRBITS;
   4445       UInt      szB    : 32 - SCALARTS_N_THRBITS - 1;
   4446       UInt      isW    : 1;
   4447    } TSW; // Thread+Size+Writeness
   4448 typedef
   4449    struct {
   4450       TSW       tsw;
   4451       WordSetID locksHeldW;
   4452       RCEC*     rcec;
   4453    }
   4454    Thr_n_RCEC;
   4455 
   4456 typedef
   4457    struct OldRef {
   4458       struct OldRef *ht_next; // to link hash table nodes together.
   4459       UWord  ga; // hash_table key, == address for which we record an access.
   4460       struct OldRef *prev; // to refs older than this one
   4461       struct OldRef *next; // to refs newer that this one
   4462       UWord stamp; // allows to order (by time of access) 2 OldRef
   4463       Thr_n_RCEC acc;
   4464    }
   4465    OldRef;
   4466 
   4467 /* Returns the or->tsw as an UInt */
   4468 static inline UInt oldref_tsw (const OldRef* or)
   4469 {
   4470    return *(const UInt*)(&or->acc.tsw);
   4471 }
   4472 
   4473 /* Compare the tsw component for 2 OldRef.
   4474    Used for OldRef hashtable (which already verifies equality of the
   4475    'key' part. */
   4476 static Word cmp_oldref_tsw (const void* node1, const void* node2 )
   4477 {
   4478    const UInt tsw1 = oldref_tsw(node1);
   4479    const UInt tsw2 = oldref_tsw(node2);
   4480 
   4481    if (tsw1 < tsw2) return -1;
   4482    if (tsw1 > tsw2) return  1;
   4483    return 0;
   4484 }
   4485 
   4486 
   4487 //////////// BEGIN OldRef pool allocator
   4488 static PoolAlloc* oldref_pool_allocator;
   4489 // Note: We only allocate elements in this pool allocator, we never free them.
   4490 // We stop allocating elements at VG_(clo_conflict_cache_size).
   4491 //////////// END OldRef pool allocator
   4492 
   4493 static OldRef mru;
   4494 static OldRef lru;
   4495 // A double linked list, chaining all OldREf in a mru/lru order.
   4496 // mru/lru are sentinel nodes.
   4497 // Whenever an oldref is re-used, its position is changed as the most recently
   4498 // used (i.e. pointed to by mru.prev).
   4499 // When a new oldref is needed, it is allocated from the pool
   4500 //  if we have not yet reached --conflict-cache-size.
   4501 // Otherwise, if all oldref have already been allocated,
   4502 // the least recently used (i.e. pointed to by lru.next) is re-used.
   4503 // When an OldRef is used, it is moved as the most recently used entry
   4504 // (i.e. pointed to by mru.prev).
   4505 
   4506 // Removes r from the double linked list
   4507 // Note: we do not need to test for special cases such as
   4508 // NULL next or prev pointers, because we have sentinel nodes
   4509 // at both sides of the list. So, a node is always forward and
   4510 // backward linked.
   4511 static inline void OldRef_unchain(OldRef *r)
   4512 {
   4513    r->next->prev = r->prev;
   4514    r->prev->next = r->next;
   4515 }
   4516 
   4517 // Insert new as the newest OldRef
   4518 // Similarly to OldRef_unchain, no need to test for NULL
   4519 // pointers, as e.g. mru.prev is always guaranteed to point
   4520 // to a non NULL node (lru when the list is empty).
   4521 static inline void OldRef_newest(OldRef *new)
   4522 {
   4523    new->next = &mru;
   4524    new->prev = mru.prev;
   4525    mru.prev = new;
   4526    new->prev->next = new;
   4527 }
   4528 
   4529 
   4530 static VgHashTable* oldrefHT    = NULL; /* Hash table* OldRef* */
   4531 static UWord     oldrefHTN    = 0;    /* # elems in oldrefHT */
   4532 /* Note: the nr of ref in the oldrefHT will always be equal to
   4533    the nr of elements that were allocated from the OldRef pool allocator
   4534    as we never free an OldRef : we just re-use them. */
   4535 
   4536 
   4537 /* allocates a new OldRef or re-use the lru one if all allowed OldRef
   4538    have already been allocated. */
   4539 static OldRef* alloc_or_reuse_OldRef ( void )
   4540 {
   4541    if (oldrefHTN < HG_(clo_conflict_cache_size)) {
   4542       oldrefHTN++;
   4543       return VG_(allocEltPA) ( oldref_pool_allocator );
   4544    } else {
   4545       OldRef *oldref_ht;
   4546       OldRef *oldref = lru.next;
   4547 
   4548       OldRef_unchain(oldref);
   4549       oldref_ht = VG_(HT_gen_remove) (oldrefHT, oldref, cmp_oldref_tsw);
   4550       tl_assert (oldref == oldref_ht);
   4551       ctxt__rcdec( oldref->acc.rcec );
   4552       return oldref;
   4553    }
   4554 }
   4555 
   4556 
   4557 inline static UInt min_UInt ( UInt a, UInt b ) {
   4558    return a < b ? a : b;
   4559 }
   4560 
   4561 /* Compare the intervals [a1,a1+n1) and [a2,a2+n2).  Return -1 if the
   4562    first interval is lower, 1 if the first interval is higher, and 0
   4563    if there is any overlap.  Redundant paranoia with casting is there
   4564    following what looked distinctly like a bug in gcc-4.1.2, in which
   4565    some of the comparisons were done signedly instead of
   4566    unsignedly. */
   4567 /* Copied from exp-ptrcheck/sg_main.c */
   4568 static inline Word cmp_nonempty_intervals ( Addr a1, SizeT n1,
   4569                                             Addr a2, SizeT n2 ) {
   4570    UWord a1w = (UWord)a1;
   4571    UWord n1w = (UWord)n1;
   4572    UWord a2w = (UWord)a2;
   4573    UWord n2w = (UWord)n2;
   4574    tl_assert(n1w > 0 && n2w > 0);
   4575    if (a1w + n1w <= a2w) return -1L;
   4576    if (a2w + n2w <= a1w) return 1L;
   4577    return 0;
   4578 }
   4579 
   4580 static UWord event_map_stamp = 0; // Used to stamp each OldRef when touched.
   4581 
   4582 static void event_map_bind ( Addr a, SizeT szB, Bool isW, Thr* thr )
   4583 {
   4584    OldRef  example;
   4585    OldRef* ref;
   4586    RCEC*   rcec;
   4587 
   4588    tl_assert(thr);
   4589    ThrID thrid = thr->thrid;
   4590    tl_assert(thrid != 0); /* zero is used to denote an empty slot. */
   4591 
   4592    WordSetID locksHeldW = thr->hgthread->locksetW;
   4593 
   4594    rcec = get_RCEC( thr );
   4595 
   4596    tl_assert (szB == 4 || szB == 8 ||szB == 1 || szB == 2);
   4597    // Check for most frequent cases first
   4598    // Note: we could support a szB up to 1 << (32 - SCALARTS_N_THRBITS - 1)
   4599 
   4600    /* Look in the oldrefHT to see if we already have a record for this
   4601       address/thr/sz/isW. */
   4602    example.ga = a;
   4603    example.acc.tsw = (TSW) {.thrid = thrid,
   4604                             .szB = szB,
   4605                             .isW = (UInt)(isW & 1)};
   4606    ref = VG_(HT_gen_lookup) (oldrefHT, &example, cmp_oldref_tsw);
   4607 
   4608    if (ref) {
   4609       /* We already have a record for this address and this (thrid, R/W,
   4610          size) triple. */
   4611       tl_assert (ref->ga == a);
   4612 
   4613       /* thread 'thr' has an entry.  Update its RCEC, if it differs. */
   4614       if (rcec == ref->acc.rcec)
   4615          stats__ctxt_eq_tsw_eq_rcec++;
   4616       else {
   4617          stats__ctxt_eq_tsw_neq_rcec++;
   4618          ctxt__rcdec( ref->acc.rcec );
   4619          ctxt__rcinc(rcec);
   4620          ref->acc.rcec       = rcec;
   4621       }
   4622       tl_assert(ref->acc.tsw.thrid == thrid);
   4623       /* Update the stamp, RCEC and the W-held lockset. */
   4624       ref->stamp = event_map_stamp;
   4625       ref->acc.locksHeldW = locksHeldW;
   4626 
   4627       OldRef_unchain(ref);
   4628       OldRef_newest(ref);
   4629 
   4630    } else {
   4631       /* We don't have a record for this address+triple.  Create a new one. */
   4632       stats__ctxt_neq_tsw_neq_rcec++;
   4633       ref = alloc_or_reuse_OldRef();
   4634       ref->ga = a;
   4635       ref->acc.tsw = (TSW) {.thrid  = thrid,
   4636                             .szB    = szB,
   4637                             .isW    = (UInt)(isW & 1)};
   4638       ref->stamp = event_map_stamp;
   4639       ref->acc.locksHeldW = locksHeldW;
   4640       ref->acc.rcec       = rcec;
   4641       ctxt__rcinc(rcec);
   4642 
   4643       VG_(HT_add_node) ( oldrefHT, ref );
   4644       OldRef_newest (ref);
   4645    }
   4646    event_map_stamp++;
   4647 }
   4648 
   4649 
   4650 /* Extract info from the conflicting-access machinery.
   4651    Returns the most recent conflicting access with thr/[a, a+szB[/isW. */
   4652 Bool libhb_event_map_lookup ( /*OUT*/ExeContext** resEC,
   4653                               /*OUT*/Thr**        resThr,
   4654                               /*OUT*/SizeT*       resSzB,
   4655                               /*OUT*/Bool*        resIsW,
   4656                               /*OUT*/WordSetID*   locksHeldW,
   4657                               Thr* thr, Addr a, SizeT szB, Bool isW )
   4658 {
   4659    Word    i, j;
   4660    OldRef *ref = NULL;
   4661    SizeT  ref_szB = 0;
   4662 
   4663    OldRef *cand_ref;
   4664    SizeT  cand_ref_szB;
   4665    Addr   cand_a;
   4666 
   4667    Addr toCheck[15];
   4668    Int  nToCheck = 0;
   4669 
   4670    tl_assert(thr);
   4671    tl_assert(szB == 8 || szB == 4 || szB == 2 || szB == 1);
   4672 
   4673    ThrID thrid = thr->thrid;
   4674 
   4675    toCheck[nToCheck++] = a;
   4676    for (i = -7; i < (Word)szB; i++) {
   4677       if (i != 0)
   4678          toCheck[nToCheck++] = a + i;
   4679    }
   4680    tl_assert(nToCheck <= 15);
   4681 
   4682    /* Now see if we can find a suitable matching event for
   4683       any of the addresses in toCheck[0 .. nToCheck-1]. */
   4684    for (j = 0; j < nToCheck; j++) {
   4685 
   4686       cand_a = toCheck[j];
   4687       //      VG_(printf)("test %ld %p\n", j, cand_a);
   4688 
   4689       /* Find the first HT element for this address.
   4690          We might have several of these. They will be linked via ht_next.
   4691          We however need to check various elements as the list contains
   4692          all elements that map to the same bucket. */
   4693       for (cand_ref = VG_(HT_lookup)( oldrefHT, cand_a );
   4694            cand_ref; cand_ref = cand_ref->ht_next) {
   4695          if (cand_ref->ga != cand_a)
   4696             /* OldRef for another address in this HT bucket. Ignore. */
   4697             continue;
   4698 
   4699          if (cand_ref->acc.tsw.thrid == thrid)
   4700             /* This is an access by the same thread, but we're only
   4701                interested in accesses from other threads.  Ignore. */
   4702             continue;
   4703 
   4704          if ((!cand_ref->acc.tsw.isW) && (!isW))
   4705             /* We don't want to report a read racing against another
   4706                read; that's stupid.  So in this case move on. */
   4707             continue;
   4708 
   4709          cand_ref_szB        = cand_ref->acc.tsw.szB;
   4710          if (cmp_nonempty_intervals(a, szB, cand_a, cand_ref_szB) != 0)
   4711             /* No overlap with the access we're asking about.  Ignore. */
   4712             continue;
   4713 
   4714          /* We have a match. Keep this match if it is newer than
   4715             the previous match. Note that stamp are Unsigned Words, and
   4716             for long running applications, event_map_stamp might have cycled.
   4717             So, 'roll' each stamp using event_map_stamp to have the
   4718             stamps in the good order, in case event_map_stamp recycled. */
   4719          if (!ref
   4720              || (ref->stamp - event_map_stamp)
   4721                    < (cand_ref->stamp - event_map_stamp)) {
   4722             ref = cand_ref;
   4723             ref_szB = cand_ref_szB;
   4724          }
   4725       }
   4726 
   4727       if (ref) {
   4728          /* return with success */
   4729          Int n, maxNFrames;
   4730          RCEC*     ref_rcec = ref->acc.rcec;
   4731          tl_assert(ref->acc.tsw.thrid);
   4732          tl_assert(ref_rcec);
   4733          tl_assert(ref_rcec->magic == RCEC_MAGIC);
   4734          tl_assert(ref_szB >= 1);
   4735          /* Count how many non-zero frames we have. */
   4736          maxNFrames = min_UInt(N_FRAMES, VG_(clo_backtrace_size));
   4737          for (n = 0; n < maxNFrames; n++) {
   4738             if (0 == ref_rcec->frames[n]) break;
   4739          }
   4740          *resEC      = VG_(make_ExeContext_from_StackTrace)(ref_rcec->frames,
   4741                                                             n);
   4742          *resThr     = Thr__from_ThrID(ref->acc.tsw.thrid);
   4743          *resSzB     = ref_szB;
   4744          *resIsW     = ref->acc.tsw.isW;
   4745          *locksHeldW = ref->acc.locksHeldW;
   4746          stats__evm__lookup_found++;
   4747          return True;
   4748       }
   4749 
   4750       /* consider next address in toCheck[] */
   4751    } /* for (j = 0; j < nToCheck; j++) */
   4752 
   4753    /* really didn't find anything. */
   4754    stats__evm__lookup_notfound++;
   4755    return False;
   4756 }
   4757 
   4758 
   4759 void libhb_event_map_access_history ( Addr a, SizeT szB, Access_t fn )
   4760 {
   4761    OldRef *ref = lru.next;
   4762    SizeT ref_szB;
   4763    Int n;
   4764 
   4765    while (ref != &mru) {
   4766       ref_szB = ref->acc.tsw.szB;
   4767       if (cmp_nonempty_intervals(a, szB, ref->ga, ref_szB) == 0) {
   4768          RCEC* ref_rcec = ref->acc.rcec;
   4769          for (n = 0; n < N_FRAMES; n++) {
   4770             if (0 == ref_rcec->frames[n]) {
   4771                break;
   4772             }
   4773          }
   4774          (*fn)(ref_rcec->frames, n,
   4775                Thr__from_ThrID(ref->acc.tsw.thrid),
   4776                ref->ga,
   4777                ref_szB,
   4778                ref->acc.tsw.isW,
   4779                ref->acc.locksHeldW);
   4780       }
   4781       tl_assert (ref->next == &mru
   4782                  || ((ref->stamp - event_map_stamp)
   4783                         < ref->next->stamp - event_map_stamp));
   4784       ref = ref->next;
   4785    }
   4786 }
   4787 
   4788 static void event_map_init ( void )
   4789 {
   4790    Word i;
   4791 
   4792    /* Context (RCEC) pool allocator */
   4793    rcec_pool_allocator = VG_(newPA) (
   4794                              sizeof(RCEC),
   4795                              1000 /* RCECs per pool */,
   4796                              HG_(zalloc),
   4797                              "libhb.event_map_init.1 (RCEC pools)",
   4798                              HG_(free)
   4799                           );
   4800 
   4801    /* Context table */
   4802    tl_assert(!contextTab);
   4803    contextTab = HG_(zalloc)( "libhb.event_map_init.2 (context table)",
   4804                              N_RCEC_TAB * sizeof(RCEC*) );
   4805    for (i = 0; i < N_RCEC_TAB; i++)
   4806       contextTab[i] = NULL;
   4807 
   4808    /* Oldref pool allocator */
   4809    oldref_pool_allocator = VG_(newPA)(
   4810                                sizeof(OldRef),
   4811                                1000 /* OldRefs per pool */,
   4812                                HG_(zalloc),
   4813                                "libhb.event_map_init.3 (OldRef pools)",
   4814                                HG_(free)
   4815                             );
   4816 
   4817    /* Oldref hashtable */
   4818    tl_assert(!oldrefHT);
   4819    oldrefHT = VG_(HT_construct) ("libhb.event_map_init.4 (oldref hashtable)");
   4820 
   4821    oldrefHTN = 0;
   4822    mru.prev = &lru;
   4823    mru.next = NULL;
   4824    lru.prev = NULL;
   4825    lru.next = &mru;
   4826    mru.acc = (Thr_n_RCEC) {.tsw = {.thrid = 0,
   4827                                    .szB = 0,
   4828                                    .isW = 0},
   4829                            .locksHeldW = 0,
   4830                            .rcec = NULL};
   4831    lru.acc = mru.acc;
   4832 }
   4833 
   4834 static void event_map__check_reference_counts ( void )
   4835 {
   4836    RCEC*   rcec;
   4837    OldRef* oldref;
   4838    Word    i;
   4839    UWord   nEnts = 0;
   4840 
   4841    /* Set the 'check' reference counts to zero.  Also, optionally
   4842       check that the real reference counts are non-zero.  We allow
   4843       these to fall to zero before a GC, but the GC must get rid of
   4844       all those that are zero, hence none should be zero after a
   4845       GC. */
   4846    for (i = 0; i < N_RCEC_TAB; i++) {
   4847       for (rcec = contextTab[i]; rcec; rcec = rcec->next) {
   4848          nEnts++;
   4849          tl_assert(rcec);
   4850          tl_assert(rcec->magic == RCEC_MAGIC);
   4851          rcec->rcX = 0;
   4852       }
   4853    }
   4854 
   4855    /* check that the stats are sane */
   4856    tl_assert(nEnts == stats__ctxt_tab_curr);
   4857    tl_assert(stats__ctxt_tab_curr <= stats__ctxt_tab_max);
   4858 
   4859    /* visit all the referencing points, inc check ref counts */
   4860    VG_(HT_ResetIter)( oldrefHT );
   4861    oldref = VG_(HT_Next)( oldrefHT );
   4862    while (oldref) {
   4863       tl_assert (oldref->acc.tsw.thrid);
   4864       tl_assert (oldref->acc.rcec);
   4865       tl_assert (oldref->acc.rcec->magic == RCEC_MAGIC);
   4866       oldref->acc.rcec->rcX++;
   4867       oldref = VG_(HT_Next)( oldrefHT );
   4868    }
   4869 
   4870    /* compare check ref counts with actual */
   4871    for (i = 0; i < N_RCEC_TAB; i++) {
   4872       for (rcec = contextTab[i]; rcec; rcec = rcec->next) {
   4873          tl_assert(rcec->rc == rcec->rcX);
   4874       }
   4875    }
   4876 }
   4877 
   4878 __attribute__((noinline))
   4879 static void do_RCEC_GC ( void )
   4880 {
   4881    UInt i;
   4882 
   4883    if (VG_(clo_stats)) {
   4884       static UInt ctr = 1;
   4885       VG_(message)(Vg_DebugMsg,
   4886                   "libhb: RCEC GC: #%u  %lu slots,"
   4887                    " %lu cur ents(ref'd %lu),"
   4888                    " %lu max ents\n",
   4889                    ctr++,
   4890                    (UWord)N_RCEC_TAB,
   4891                    stats__ctxt_tab_curr, RCEC_referenced,
   4892                    stats__ctxt_tab_max );
   4893    }
   4894    tl_assert (stats__ctxt_tab_curr > RCEC_referenced);
   4895 
   4896    /* Throw away all RCECs with zero reference counts */
   4897    for (i = 0; i < N_RCEC_TAB; i++) {
   4898       RCEC** pp = &contextTab[i];
   4899       RCEC*  p  = *pp;
   4900       while (p) {
   4901          if (p->rc == 0) {
   4902             *pp = p->next;
   4903             free_RCEC(p);
   4904             p = *pp;
   4905             tl_assert(stats__ctxt_tab_curr > 0);
   4906             stats__ctxt_rcec_gc_discards++;
   4907             stats__ctxt_tab_curr--;
   4908          } else {
   4909             pp = &p->next;
   4910             p = p->next;
   4911          }
   4912       }
   4913    }
   4914 
   4915    tl_assert (stats__ctxt_tab_curr == RCEC_referenced);
   4916 }
   4917 
   4918 /////////////////////////////////////////////////////////
   4919 //                                                     //
   4920 // Core MSM                                            //
   4921 //                                                     //
   4922 /////////////////////////////////////////////////////////
   4923 
   4924 /* Logic in msmcread/msmcwrite updated/verified after re-analysis, 19
   4925    Nov 08, and again after [...],
   4926    June 09. */
   4927 
   4928 static ULong stats__msmcread         = 0;
   4929 static ULong stats__msmcread_change  = 0;
   4930 static ULong stats__msmcwrite        = 0;
   4931 static ULong stats__msmcwrite_change = 0;
   4932 
   4933 /* Some notes on the H1 history mechanism:
   4934 
   4935    Transition rules are:
   4936 
   4937    read_{Kr,Kw}(Cr,Cw)  = (Cr,           Cr `join` Kw)
   4938    write_{Kr,Kw}(Cr,Cw) = (Cr `join` Kw, Cr `join` Kw)
   4939 
   4940    After any access by a thread T to a location L, L's constraint pair
   4941    (Cr,Cw) has Cw[T] == T's Kw[T], that is, == T's scalar W-clock.
   4942 
   4943    After a race by thread T conflicting with some previous access by
   4944    some other thread U, for a location with constraint (before
   4945    processing the later access) (Cr,Cw), then Cw[U] is the segment in
   4946    which the previously access lies.
   4947 
   4948    Hence in record_race_info, we pass in Cfailed and Kfailed, which
   4949    are compared so as to find out which thread(s) this access
   4950    conflicts with.  Once that is established, we also require the
   4951    pre-update Cw for the location, so we can index into it for those
   4952    threads, to get the scalar clock values for the point at which the
   4953    former accesses were made.  (In fact we only bother to do any of
   4954    this for an arbitrarily chosen one of the conflicting threads, as
   4955    that's simpler, it avoids flooding the user with vast amounts of
   4956    mostly useless information, and because the program is wrong if it
   4957    contains any races at all -- so we don't really need to show all
   4958    conflicting access pairs initially, so long as we only show none if
   4959    none exist).
   4960 
   4961    ---
   4962 
   4963    That requires the auxiliary proof that
   4964 
   4965       (Cr `join` Kw)[T] == Kw[T]
   4966 
   4967    Why should that be true?  Because for any thread T, Kw[T] >= the
   4968    scalar clock value for T known by any other thread.  In other
   4969    words, because T's value for its own scalar clock is at least as up
   4970    to date as the value for it known by any other thread (that is true
   4971    for both the R- and W- scalar clocks).  Hence no other thread will
   4972    be able to feed in a value for that element (indirectly via a
   4973    constraint) which will exceed Kw[T], and hence the join cannot
   4974    cause that particular element to advance.
   4975 */
   4976 
   4977 __attribute__((noinline))
   4978 static void record_race_info ( Thr* acc_thr,
   4979                                Addr acc_addr, SizeT szB, Bool isWrite,
   4980                                VtsID Cfailed,
   4981                                VtsID Kfailed,
   4982                                VtsID Cw )
   4983 {
   4984    /* Call here to report a race.  We just hand it onwards to
   4985       HG_(record_error_Race).  If that in turn discovers that the
   4986       error is going to be collected, then, at history_level 2, that
   4987       queries the conflicting-event map.  The alternative would be to
   4988       query it right here.  But that causes a lot of pointless queries
   4989       for errors which will shortly be discarded as duplicates, and
   4990       can become a performance overhead; so we defer the query until
   4991       we know the error is not a duplicate. */
   4992 
   4993    /* Stacks for the bounds of the (or one of the) conflicting
   4994       segment(s).  These are only set at history_level 1. */
   4995    ExeContext* hist1_seg_start = NULL;
   4996    ExeContext* hist1_seg_end   = NULL;
   4997    Thread*     hist1_conf_thr  = NULL;
   4998 
   4999    tl_assert(acc_thr);
   5000    tl_assert(acc_thr->hgthread);
   5001    tl_assert(acc_thr->hgthread->hbthr == acc_thr);
   5002    tl_assert(HG_(clo_history_level) >= 0 && HG_(clo_history_level) <= 2);
   5003 
   5004    if (HG_(clo_history_level) == 1) {
   5005       Bool found;
   5006       Word firstIx, lastIx;
   5007       ULong_n_EC key;
   5008 
   5009       /* At history_level 1, we must round up the relevant stack-pair
   5010          for the conflicting segment right now.  This is because
   5011          deferring it is complex; we can't (easily) put Kfailed and
   5012          Cfailed into the XError and wait for later without
   5013          getting tied up in difficulties with VtsID reference
   5014          counting.  So just do it now. */
   5015       Thr*  confThr;
   5016       ULong confTym = 0;
   5017       /* Which thread are we in conflict with?  There may be more than
   5018          one, in which case VtsID__findFirst_notLEQ selects one arbitrarily
   5019          (in fact it's the one with the lowest Thr* value). */
   5020       confThr = VtsID__findFirst_notLEQ( Cfailed, Kfailed );
   5021       /* This must exist!  since if it was NULL then there's no
   5022          conflict (semantics of return value of
   5023          VtsID__findFirst_notLEQ), and msmc{read,write}, which has
   5024          called us, just checked exactly this -- that there was in
   5025          fact a race. */
   5026       tl_assert(confThr);
   5027 
   5028       /* Get the scalar clock value that the conflicting thread
   5029          introduced into the constraint.  A careful examination of the
   5030          base machine rules shows that this must be the same as the
   5031          conflicting thread's scalar clock when it created this
   5032          constraint.  Hence we know the scalar clock of the
   5033          conflicting thread when the conflicting access was made. */
   5034       confTym = VtsID__indexAt( Cfailed, confThr );
   5035 
   5036       /* Using this scalar clock, index into the conflicting thread's
   5037          collection of stack traces made each time its vector clock
   5038          (hence its scalar clock) changed.  This gives the stack
   5039          traces at the start and end of the conflicting segment (well,
   5040          as per comment just above, of one of the conflicting
   5041          segments, if there are more than one). */
   5042       key.ull = confTym;
   5043       key.ec  = NULL;
   5044       /* tl_assert(confThr); -- asserted just above */
   5045       tl_assert(confThr->local_Kws_n_stacks);
   5046       firstIx = lastIx = 0;
   5047       found = VG_(lookupXA_UNSAFE)(
   5048                  confThr->local_Kws_n_stacks,
   5049                  &key, &firstIx, &lastIx,
   5050                  (XACmpFn_t)cmp__ULong_n_EC__by_ULong
   5051               );
   5052       if (0) VG_(printf)("record_race_info %u %u %u  confThr %p "
   5053                          "confTym %llu found %d (%ld,%ld)\n",
   5054                          Cfailed, Kfailed, Cw,
   5055                          confThr, confTym, found, firstIx, lastIx);
   5056       /* We can't indefinitely collect stack traces at VTS
   5057          transitions, since we'd eventually run out of memory.  Hence
   5058          note_local_Kw_n_stack_for will eventually throw away old
   5059          ones, which in turn means we might fail to find index value
   5060          confTym in the array. */
   5061       if (found) {
   5062          ULong_n_EC *pair_start, *pair_end;
   5063          pair_start
   5064             = (ULong_n_EC*)VG_(indexXA)( confThr->local_Kws_n_stacks, lastIx );
   5065          hist1_seg_start = pair_start->ec;
   5066          if (lastIx+1 < VG_(sizeXA)( confThr->local_Kws_n_stacks )) {
   5067             pair_end
   5068                = (ULong_n_EC*)VG_(indexXA)( confThr->local_Kws_n_stacks,
   5069                                             lastIx+1 );
   5070             /* from properties of VG_(lookupXA) and the comparison fn used: */
   5071             tl_assert(pair_start->ull < pair_end->ull);
   5072             hist1_seg_end = pair_end->ec;
   5073             /* Could do a bit better here.  It may be that pair_end
   5074                doesn't have a stack, but the following entries in the
   5075                array have the same scalar Kw and to have a stack.  So
   5076                we should search a bit further along the array than
   5077                lastIx+1 if hist1_seg_end is NULL. */
   5078          } else {
   5079             if (!confThr->llexit_done)
   5080                hist1_seg_end = main_get_EC( confThr );
   5081          }
   5082          // seg_start could be NULL iff this is the first stack in the thread
   5083          //if (seg_start) VG_(pp_ExeContext)(seg_start);
   5084          //if (seg_end)   VG_(pp_ExeContext)(seg_end);
   5085          hist1_conf_thr = confThr->hgthread;
   5086       }
   5087    }
   5088 
   5089    HG_(record_error_Race)( acc_thr->hgthread, acc_addr,
   5090                            szB, isWrite,
   5091                            hist1_conf_thr, hist1_seg_start, hist1_seg_end );
   5092 }
   5093 
   5094 static Bool is_sane_SVal_C ( SVal sv ) {
   5095    Bool leq;
   5096    if (!SVal__isC(sv)) return True;
   5097    leq = VtsID__cmpLEQ( SVal__unC_Rmin(sv), SVal__unC_Wmin(sv) );
   5098    return leq;
   5099 }
   5100 
   5101 
   5102 /* Compute new state following a read */
   5103 static inline SVal msmcread ( SVal svOld,
   5104                               /* The following are only needed for
   5105                                  creating error reports. */
   5106                               Thr* acc_thr,
   5107                               Addr acc_addr, SizeT szB )
   5108 {
   5109    SVal svNew = SVal_INVALID;
   5110    stats__msmcread++;
   5111 
   5112    /* Redundant sanity check on the constraints */
   5113    if (CHECK_MSM) {
   5114       tl_assert(is_sane_SVal_C(svOld));
   5115    }
   5116 
   5117    if (LIKELY(SVal__isC(svOld))) {
   5118       VtsID tviR  = acc_thr->viR;
   5119       VtsID tviW  = acc_thr->viW;
   5120       VtsID rmini = SVal__unC_Rmin(svOld);
   5121       VtsID wmini = SVal__unC_Wmin(svOld);
   5122       Bool  leq   = VtsID__cmpLEQ(rmini,tviR);
   5123       if (LIKELY(leq)) {
   5124          /* no race */
   5125          /* Note: RWLOCK subtlety: use tviW, not tviR */
   5126          svNew = SVal__mkC( rmini, VtsID__join2(wmini, tviW) );
   5127          goto out;
   5128       } else {
   5129          /* assert on sanity of constraints. */
   5130          Bool leqxx = VtsID__cmpLEQ(rmini,wmini);
   5131          tl_assert(leqxx);
   5132          // same as in non-race case
   5133          svNew = SVal__mkC( rmini, VtsID__join2(wmini, tviW) );
   5134          record_race_info( acc_thr, acc_addr, szB, False/*!isWrite*/,
   5135                            rmini, /* Cfailed */
   5136                            tviR,  /* Kfailed */
   5137                            wmini  /* Cw */ );
   5138          goto out;
   5139       }
   5140    }
   5141    if (SVal__isA(svOld)) {
   5142       /* reading no-access memory (sigh); leave unchanged */
   5143       /* check for no pollution */
   5144       tl_assert(svOld == SVal_NOACCESS);
   5145       svNew = SVal_NOACCESS;
   5146       goto out;
   5147    }
   5148    if (0) VG_(printf)("msmcread: bad svOld: 0x%016llx\n", svOld);
   5149    tl_assert(0);
   5150 
   5151   out:
   5152    if (CHECK_MSM) {
   5153       tl_assert(is_sane_SVal_C(svNew));
   5154    }
   5155    if (UNLIKELY(svNew != svOld)) {
   5156       tl_assert(svNew != SVal_INVALID);
   5157       if (HG_(clo_history_level) >= 2
   5158           && SVal__isC(svOld) && SVal__isC(svNew)) {
   5159          event_map_bind( acc_addr, szB, False/*!isWrite*/, acc_thr );
   5160          stats__msmcread_change++;
   5161       }
   5162    }
   5163    return svNew;
   5164 }
   5165 
   5166 
   5167 /* Compute new state following a write */
   5168 static inline SVal msmcwrite ( SVal svOld,
   5169                               /* The following are only needed for
   5170                                  creating error reports. */
   5171                               Thr* acc_thr,
   5172                               Addr acc_addr, SizeT szB )
   5173 {
   5174    SVal svNew = SVal_INVALID;
   5175    stats__msmcwrite++;
   5176 
   5177    /* Redundant sanity check on the constraints */
   5178    if (CHECK_MSM) {
   5179       tl_assert(is_sane_SVal_C(svOld));
   5180    }
   5181 
   5182    if (LIKELY(SVal__isC(svOld))) {
   5183       VtsID tviW  = acc_thr->viW;
   5184       VtsID wmini = SVal__unC_Wmin(svOld);
   5185       Bool  leq   = VtsID__cmpLEQ(wmini,tviW);
   5186       if (LIKELY(leq)) {
   5187          /* no race */
   5188          svNew = SVal__mkC( tviW, tviW );
   5189          goto out;
   5190       } else {
   5191          VtsID rmini = SVal__unC_Rmin(svOld);
   5192          /* assert on sanity of constraints. */
   5193          Bool leqxx = VtsID__cmpLEQ(rmini,wmini);
   5194          tl_assert(leqxx);
   5195          // same as in non-race case
   5196          // proof: in the non-race case, we have
   5197          //    rmini <= wmini (invar on constraints)
   5198          //    tviW <= tviR (invar on thread clocks)
   5199          //    wmini <= tviW (from run-time check)
   5200          // hence from transitivity of <= we have
   5201          //    rmini <= wmini <= tviW
   5202          // and so join(rmini,tviW) == tviW
   5203          // and    join(wmini,tviW) == tviW
   5204          // qed.
   5205          svNew = SVal__mkC( VtsID__join2(rmini, tviW),
   5206                             VtsID__join2(wmini, tviW) );
   5207          record_race_info( acc_thr, acc_addr, szB, True/*isWrite*/,
   5208                            wmini, /* Cfailed */
   5209                            tviW,  /* Kfailed */
   5210                            wmini  /* Cw */ );
   5211          goto out;
   5212       }
   5213    }
   5214    if (SVal__isA(svOld)) {
   5215       /* writing no-access memory (sigh); leave unchanged */
   5216       /* check for no pollution */
   5217       tl_assert(svOld == SVal_NOACCESS);
   5218       svNew = SVal_NOACCESS;
   5219       goto out;
   5220    }
   5221    if (0) VG_(printf)("msmcwrite: bad svOld: 0x%016llx\n", svOld);
   5222    tl_assert(0);
   5223 
   5224   out:
   5225    if (CHECK_MSM) {
   5226       tl_assert(is_sane_SVal_C(svNew));
   5227    }
   5228    if (UNLIKELY(svNew != svOld)) {
   5229       tl_assert(svNew != SVal_INVALID);
   5230       if (HG_(clo_history_level) >= 2
   5231           && SVal__isC(svOld) && SVal__isC(svNew)) {
   5232          event_map_bind( acc_addr, szB, True/*isWrite*/, acc_thr );
   5233          stats__msmcwrite_change++;
   5234       }
   5235    }
   5236    return svNew;
   5237 }
   5238 
   5239 
   5240 /////////////////////////////////////////////////////////
   5241 //                                                     //
   5242 // Apply core MSM to specific memory locations         //
   5243 //                                                     //
   5244 /////////////////////////////////////////////////////////
   5245 
   5246 /*------------- ZSM accesses: 8 bit sapply ------------- */
   5247 
   5248 static void zsm_sapply08__msmcread ( Thr* thr, Addr a ) {
   5249    CacheLine* cl;
   5250    UWord      cloff, tno, toff;
   5251    SVal       svOld, svNew;
   5252    UShort     descr;
   5253    stats__cline_cread08s++;
   5254    cl    = get_cacheline(a);
   5255    cloff = get_cacheline_offset(a);
   5256    tno   = get_treeno(a);
   5257    toff  = get_tree_offset(a); /* == 0 .. 7 */
   5258    descr = cl->descrs[tno];
   5259    if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) {
   5260       SVal* tree = &cl->svals[tno << 3];
   5261       cl->descrs[tno] = pulldown_to_8(tree, toff, descr);
   5262       if (CHECK_ZSM)
   5263          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5264    }
   5265    svOld = cl->svals[cloff];
   5266    svNew = msmcread( svOld, thr,a,1 );
   5267    if (CHECK_ZSM)
   5268       tl_assert(svNew != SVal_INVALID);
   5269    cl->svals[cloff] = svNew;
   5270 }
   5271 
   5272 static void zsm_sapply08__msmcwrite ( Thr* thr, Addr a ) {
   5273    CacheLine* cl;
   5274    UWord      cloff, tno, toff;
   5275    SVal       svOld, svNew;
   5276    UShort     descr;
   5277    stats__cline_cwrite08s++;
   5278    cl    = get_cacheline(a);
   5279    cloff = get_cacheline_offset(a);
   5280    tno   = get_treeno(a);
   5281    toff  = get_tree_offset(a); /* == 0 .. 7 */
   5282    descr = cl->descrs[tno];
   5283    if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) {
   5284       SVal* tree = &cl->svals[tno << 3];
   5285       cl->descrs[tno] = pulldown_to_8(tree, toff, descr);
   5286       if (CHECK_ZSM)
   5287          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5288    }
   5289    svOld = cl->svals[cloff];
   5290    svNew = msmcwrite( svOld, thr,a,1 );
   5291    if (CHECK_ZSM)
   5292       tl_assert(svNew != SVal_INVALID);
   5293    cl->svals[cloff] = svNew;
   5294 }
   5295 
   5296 /*------------- ZSM accesses: 16 bit sapply ------------- */
   5297 
   5298 static void zsm_sapply16__msmcread ( Thr* thr, Addr a ) {
   5299    CacheLine* cl;
   5300    UWord      cloff, tno, toff;
   5301    SVal       svOld, svNew;
   5302    UShort     descr;
   5303    stats__cline_cread16s++;
   5304    if (UNLIKELY(!aligned16(a))) goto slowcase;
   5305    cl    = get_cacheline(a);
   5306    cloff = get_cacheline_offset(a);
   5307    tno   = get_treeno(a);
   5308    toff  = get_tree_offset(a); /* == 0, 2, 4 or 6 */
   5309    descr = cl->descrs[tno];
   5310    if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) {
   5311       if (valid_value_is_below_me_16(descr, toff)) {
   5312          goto slowcase;
   5313       } else {
   5314          SVal* tree = &cl->svals[tno << 3];
   5315          cl->descrs[tno] = pulldown_to_16(tree, toff, descr);
   5316       }
   5317       if (CHECK_ZSM)
   5318          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5319    }
   5320    svOld = cl->svals[cloff];
   5321    svNew = msmcread( svOld, thr,a,2 );
   5322    if (CHECK_ZSM)
   5323       tl_assert(svNew != SVal_INVALID);
   5324    cl->svals[cloff] = svNew;
   5325    return;
   5326   slowcase: /* misaligned, or must go further down the tree */
   5327    stats__cline_16to8splits++;
   5328    zsm_sapply08__msmcread( thr, a + 0 );
   5329    zsm_sapply08__msmcread( thr, a + 1 );
   5330 }
   5331 
   5332 static void zsm_sapply16__msmcwrite ( Thr* thr, Addr a ) {
   5333    CacheLine* cl;
   5334    UWord      cloff, tno, toff;
   5335    SVal       svOld, svNew;
   5336    UShort     descr;
   5337    stats__cline_cwrite16s++;
   5338    if (UNLIKELY(!aligned16(a))) goto slowcase;
   5339    cl    = get_cacheline(a);
   5340    cloff = get_cacheline_offset(a);
   5341    tno   = get_treeno(a);
   5342    toff  = get_tree_offset(a); /* == 0, 2, 4 or 6 */
   5343    descr = cl->descrs[tno];
   5344    if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) {
   5345       if (valid_value_is_below_me_16(descr, toff)) {
   5346          goto slowcase;
   5347       } else {
   5348          SVal* tree = &cl->svals[tno << 3];
   5349          cl->descrs[tno] = pulldown_to_16(tree, toff, descr);
   5350       }
   5351       if (CHECK_ZSM)
   5352          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5353    }
   5354    svOld = cl->svals[cloff];
   5355    svNew = msmcwrite( svOld, thr,a,2 );
   5356    if (CHECK_ZSM)
   5357       tl_assert(svNew != SVal_INVALID);
   5358    cl->svals[cloff] = svNew;
   5359    return;
   5360   slowcase: /* misaligned, or must go further down the tree */
   5361    stats__cline_16to8splits++;
   5362    zsm_sapply08__msmcwrite( thr, a + 0 );
   5363    zsm_sapply08__msmcwrite( thr, a + 1 );
   5364 }
   5365 
   5366 /*------------- ZSM accesses: 32 bit sapply ------------- */
   5367 
   5368 static void zsm_sapply32__msmcread ( Thr* thr, Addr a ) {
   5369    CacheLine* cl;
   5370    UWord      cloff, tno, toff;
   5371    SVal       svOld, svNew;
   5372    UShort     descr;
   5373    stats__cline_cread32s++;
   5374    if (UNLIKELY(!aligned32(a))) goto slowcase;
   5375    cl    = get_cacheline(a);
   5376    cloff = get_cacheline_offset(a);
   5377    tno   = get_treeno(a);
   5378    toff  = get_tree_offset(a); /* == 0 or 4 */
   5379    descr = cl->descrs[tno];
   5380    if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) {
   5381       if (valid_value_is_above_me_32(descr, toff)) {
   5382          SVal* tree = &cl->svals[tno << 3];
   5383          cl->descrs[tno] = pulldown_to_32(tree, toff, descr);
   5384       } else {
   5385          goto slowcase;
   5386       }
   5387       if (CHECK_ZSM)
   5388          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5389    }
   5390    svOld = cl->svals[cloff];
   5391    svNew = msmcread( svOld, thr,a,4 );
   5392    if (CHECK_ZSM)
   5393       tl_assert(svNew != SVal_INVALID);
   5394    cl->svals[cloff] = svNew;
   5395    return;
   5396   slowcase: /* misaligned, or must go further down the tree */
   5397    stats__cline_32to16splits++;
   5398    zsm_sapply16__msmcread( thr, a + 0 );
   5399    zsm_sapply16__msmcread( thr, a + 2 );
   5400 }
   5401 
   5402 static void zsm_sapply32__msmcwrite ( Thr* thr, Addr a ) {
   5403    CacheLine* cl;
   5404    UWord      cloff, tno, toff;
   5405    SVal       svOld, svNew;
   5406    UShort     descr;
   5407    stats__cline_cwrite32s++;
   5408    if (UNLIKELY(!aligned32(a))) goto slowcase;
   5409    cl    = get_cacheline(a);
   5410    cloff = get_cacheline_offset(a);
   5411    tno   = get_treeno(a);
   5412    toff  = get_tree_offset(a); /* == 0 or 4 */
   5413    descr = cl->descrs[tno];
   5414    if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) {
   5415       if (valid_value_is_above_me_32(descr, toff)) {
   5416          SVal* tree = &cl->svals[tno << 3];
   5417          cl->descrs[tno] = pulldown_to_32(tree, toff, descr);
   5418       } else {
   5419          goto slowcase;
   5420       }
   5421       if (CHECK_ZSM)
   5422          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5423    }
   5424    svOld = cl->svals[cloff];
   5425    svNew = msmcwrite( svOld, thr,a,4 );
   5426    if (CHECK_ZSM)
   5427       tl_assert(svNew != SVal_INVALID);
   5428    cl->svals[cloff] = svNew;
   5429    return;
   5430   slowcase: /* misaligned, or must go further down the tree */
   5431    stats__cline_32to16splits++;
   5432    zsm_sapply16__msmcwrite( thr, a + 0 );
   5433    zsm_sapply16__msmcwrite( thr, a + 2 );
   5434 }
   5435 
   5436 /*------------- ZSM accesses: 64 bit sapply ------------- */
   5437 
   5438 static void zsm_sapply64__msmcread ( Thr* thr, Addr a ) {
   5439    CacheLine* cl;
   5440    UWord      cloff, tno;
   5441    //UWord      toff;
   5442    SVal       svOld, svNew;
   5443    UShort     descr;
   5444    stats__cline_cread64s++;
   5445    if (UNLIKELY(!aligned64(a))) goto slowcase;
   5446    cl    = get_cacheline(a);
   5447    cloff = get_cacheline_offset(a);
   5448    tno   = get_treeno(a);
   5449    //toff  = get_tree_offset(a); /* == 0, unused */
   5450    descr = cl->descrs[tno];
   5451    if (UNLIKELY( !(descr & TREE_DESCR_64) )) {
   5452       goto slowcase;
   5453    }
   5454    svOld = cl->svals[cloff];
   5455    svNew = msmcread( svOld, thr,a,8 );
   5456    if (CHECK_ZSM)
   5457       tl_assert(svNew != SVal_INVALID);
   5458    cl->svals[cloff] = svNew;
   5459    return;
   5460   slowcase: /* misaligned, or must go further down the tree */
   5461    stats__cline_64to32splits++;
   5462    zsm_sapply32__msmcread( thr, a + 0 );
   5463    zsm_sapply32__msmcread( thr, a + 4 );
   5464 }
   5465 
   5466 static void zsm_sapply64__msmcwrite ( Thr* thr, Addr a ) {
   5467    CacheLine* cl;
   5468    UWord      cloff, tno;
   5469    //UWord      toff;
   5470    SVal       svOld, svNew;
   5471    UShort     descr;
   5472    stats__cline_cwrite64s++;
   5473    if (UNLIKELY(!aligned64(a))) goto slowcase;
   5474    cl    = get_cacheline(a);
   5475    cloff = get_cacheline_offset(a);
   5476    tno   = get_treeno(a);
   5477    //toff  = get_tree_offset(a); /* == 0, unused */
   5478    descr = cl->descrs[tno];
   5479    if (UNLIKELY( !(descr & TREE_DESCR_64) )) {
   5480       goto slowcase;
   5481    }
   5482    svOld = cl->svals[cloff];
   5483    svNew = msmcwrite( svOld, thr,a,8 );
   5484    if (CHECK_ZSM)
   5485       tl_assert(svNew != SVal_INVALID);
   5486    cl->svals[cloff] = svNew;
   5487    return;
   5488   slowcase: /* misaligned, or must go further down the tree */
   5489    stats__cline_64to32splits++;
   5490    zsm_sapply32__msmcwrite( thr, a + 0 );
   5491    zsm_sapply32__msmcwrite( thr, a + 4 );
   5492 }
   5493 
   5494 /*--------------- ZSM accesses: 8 bit swrite --------------- */
   5495 
   5496 static
   5497 void zsm_swrite08 ( Addr a, SVal svNew ) {
   5498    CacheLine* cl;
   5499    UWord      cloff, tno, toff;
   5500    UShort     descr;
   5501    stats__cline_swrite08s++;
   5502    cl    = get_cacheline(a);
   5503    cloff = get_cacheline_offset(a);
   5504    tno   = get_treeno(a);
   5505    toff  = get_tree_offset(a); /* == 0 .. 7 */
   5506    descr = cl->descrs[tno];
   5507    if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) {
   5508       SVal* tree = &cl->svals[tno << 3];
   5509       cl->descrs[tno] = pulldown_to_8(tree, toff, descr);
   5510       if (CHECK_ZSM)
   5511          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5512    }
   5513    tl_assert(svNew != SVal_INVALID);
   5514    cl->svals[cloff] = svNew;
   5515 }
   5516 
   5517 /*--------------- ZSM accesses: 16 bit swrite --------------- */
   5518 
   5519 static
   5520 void zsm_swrite16 ( Addr a, SVal svNew ) {
   5521    CacheLine* cl;
   5522    UWord      cloff, tno, toff;
   5523    UShort     descr;
   5524    stats__cline_swrite16s++;
   5525    if (UNLIKELY(!aligned16(a))) goto slowcase;
   5526    cl    = get_cacheline(a);
   5527    cloff = get_cacheline_offset(a);
   5528    tno   = get_treeno(a);
   5529    toff  = get_tree_offset(a); /* == 0, 2, 4 or 6 */
   5530    descr = cl->descrs[tno];
   5531    if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) {
   5532       if (valid_value_is_below_me_16(descr, toff)) {
   5533          /* Writing at this level.  Need to fix up 'descr'. */
   5534          cl->descrs[tno] = pullup_descr_to_16(descr, toff);
   5535          /* At this point, the tree does not match cl->descr[tno] any
   5536             more.  The assignments below will fix it up. */
   5537       } else {
   5538          /* We can't indiscriminately write on the w16 node as in the
   5539             w64 case, as that might make the node inconsistent with
   5540             its parent.  So first, pull down to this level. */
   5541          SVal* tree = &cl->svals[tno << 3];
   5542          cl->descrs[tno] = pulldown_to_16(tree, toff, descr);
   5543       if (CHECK_ZSM)
   5544          tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5545       }
   5546    }
   5547    tl_assert(svNew != SVal_INVALID);
   5548    cl->svals[cloff + 0] = svNew;
   5549    cl->svals[cloff + 1] = SVal_INVALID;
   5550    return;
   5551   slowcase: /* misaligned */
   5552    stats__cline_16to8splits++;
   5553    zsm_swrite08( a + 0, svNew );
   5554    zsm_swrite08( a + 1, svNew );
   5555 }
   5556 
   5557 /*--------------- ZSM accesses: 32 bit swrite --------------- */
   5558 
   5559 static
   5560 void zsm_swrite32 ( Addr a, SVal svNew ) {
   5561    CacheLine* cl;
   5562    UWord      cloff, tno, toff;
   5563    UShort     descr;
   5564    stats__cline_swrite32s++;
   5565    if (UNLIKELY(!aligned32(a))) goto slowcase;
   5566    cl    = get_cacheline(a);
   5567    cloff = get_cacheline_offset(a);
   5568    tno   = get_treeno(a);
   5569    toff  = get_tree_offset(a); /* == 0 or 4 */
   5570    descr = cl->descrs[tno];
   5571    if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) {
   5572       if (valid_value_is_above_me_32(descr, toff)) {
   5573          /* We can't indiscriminately write on the w32 node as in the
   5574             w64 case, as that might make the node inconsistent with
   5575             its parent.  So first, pull down to this level. */
   5576          SVal* tree = &cl->svals[tno << 3];
   5577          cl->descrs[tno] = pulldown_to_32(tree, toff, descr);
   5578          if (CHECK_ZSM)
   5579             tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */
   5580       } else {
   5581          /* Writing at this level.  Need to fix up 'descr'. */
   5582          cl->descrs[tno] = pullup_descr_to_32(descr, toff);
   5583          /* At this point, the tree does not match cl->descr[tno] any
   5584             more.  The assignments below will fix it up. */
   5585       }
   5586    }
   5587    tl_assert(svNew != SVal_INVALID);
   5588    cl->svals[cloff + 0] = svNew;
   5589    cl->svals[cloff + 1] = SVal_INVALID;
   5590    cl->svals[cloff + 2] = SVal_INVALID;
   5591    cl->svals[cloff + 3] = SVal_INVALID;
   5592    return;
   5593   slowcase: /* misaligned */
   5594    stats__cline_32to16splits++;
   5595    zsm_swrite16( a + 0, svNew );
   5596    zsm_swrite16( a + 2, svNew );
   5597 }
   5598 
   5599 /*--------------- ZSM accesses: 64 bit swrite --------------- */
   5600 
   5601 static
   5602 void zsm_swrite64 ( Addr a, SVal svNew ) {
   5603    CacheLine* cl;
   5604    UWord      cloff, tno;
   5605    //UWord    toff;
   5606    stats__cline_swrite64s++;
   5607    if (UNLIKELY(!aligned64(a))) goto slowcase;
   5608    cl    = get_cacheline(a);
   5609    cloff = get_cacheline_offset(a);
   5610    tno   = get_treeno(a);
   5611    //toff  = get_tree_offset(a); /* == 0, unused */
   5612    cl->descrs[tno] = TREE_DESCR_64;
   5613    tl_assert(svNew != SVal_INVALID);
   5614    cl->svals[cloff + 0] = svNew;
   5615    cl->svals[cloff + 1] = SVal_INVALID;
   5616    cl->svals[cloff + 2] = SVal_INVALID;
   5617    cl->svals[cloff + 3] = SVal_INVALID;
   5618    cl->svals[cloff + 4] = SVal_INVALID;
   5619    cl->svals[cloff + 5] = SVal_INVALID;
   5620    cl->svals[cloff + 6] = SVal_INVALID;
   5621    cl->svals[cloff + 7] = SVal_INVALID;
   5622    return;
   5623   slowcase: /* misaligned */
   5624    stats__cline_64to32splits++;
   5625    zsm_swrite32( a + 0, svNew );
   5626    zsm_swrite32( a + 4, svNew );
   5627 }
   5628 
   5629 /*------------- ZSM accesses: 8 bit sread/scopy ------------- */
   5630 
   5631 static
   5632 SVal zsm_sread08 ( Addr a ) {
   5633    CacheLine* cl;
   5634    UWord      cloff, tno, toff;
   5635    UShort     descr;
   5636    stats__cline_sread08s++;
   5637    cl    = get_cacheline(a);
   5638    cloff = get_cacheline_offset(a);
   5639    tno   = get_treeno(a);
   5640    toff  = get_tree_offset(a); /* == 0 .. 7 */
   5641    descr = cl->descrs[tno];
   5642    if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) {
   5643       SVal* tree = &cl->svals[tno << 3];
   5644       cl->descrs[tno] = pulldown_to_8(tree, toff, descr);
   5645    }
   5646    return cl->svals[cloff];
   5647 }
   5648 
   5649 static void zsm_scopy08 ( Addr src, Addr dst, Bool uu_normalise ) {
   5650    SVal       sv;
   5651    stats__cline_scopy08s++;
   5652    sv = zsm_sread08( src );
   5653    zsm_swrite08( dst, sv );
   5654 }
   5655 
   5656 
   5657 /* Block-copy states (needed for implementing realloc()).  Note this
   5658    doesn't change the filtering arrangements.  The caller of
   5659    zsm_scopy_range needs to attend to that. */
   5660 
   5661 static void zsm_scopy_range ( Addr src, Addr dst, SizeT len )
   5662 {
   5663    SizeT i;
   5664    if (len == 0)
   5665       return;
   5666 
   5667    /* assert for non-overlappingness */
   5668    tl_assert(src+len <= dst || dst+len <= src);
   5669 
   5670    /* To be simple, just copy byte by byte.  But so as not to wreck
   5671       performance for later accesses to dst[0 .. len-1], normalise
   5672       destination lines as we finish with them, and also normalise the
   5673       line containing the first and last address. */
   5674    for (i = 0; i < len; i++) {
   5675       Bool normalise
   5676          = get_cacheline_offset( dst+i+1 ) == 0 /* last in line */
   5677            || i == 0       /* first in range */
   5678            || i == len-1;  /* last in range */
   5679       zsm_scopy08( src+i, dst+i, normalise );
   5680    }
   5681 }
   5682 
   5683 
   5684 /* For setting address ranges to a given value.  Has considerable
   5685    sophistication so as to avoid generating large numbers of pointless
   5686    cache loads/writebacks for large ranges. */
   5687 
   5688 /* Do small ranges in-cache, in the obvious way. */
   5689 static
   5690 void zsm_sset_range_SMALL ( Addr a, SizeT len, SVal svNew )
   5691 {
   5692    /* fast track a couple of common cases */
   5693    if (len == 4 && aligned32(a)) {
   5694       zsm_swrite32( a, svNew );
   5695       return;
   5696    }
   5697    if (len == 8 && aligned64(a)) {
   5698       zsm_swrite64( a, svNew );
   5699       return;
   5700    }
   5701 
   5702    /* be completely general (but as efficient as possible) */
   5703    if (len == 0) return;
   5704 
   5705    if (!aligned16(a) && len >= 1) {
   5706       zsm_swrite08( a, svNew );
   5707       a += 1;
   5708       len -= 1;
   5709       tl_assert(aligned16(a));
   5710    }
   5711    if (len == 0) return;
   5712 
   5713    if (!aligned32(a) && len >= 2) {
   5714       zsm_swrite16( a, svNew );
   5715       a += 2;
   5716       len -= 2;
   5717       tl_assert(aligned32(a));
   5718    }
   5719    if (len == 0) return;
   5720 
   5721    if (!aligned64(a) && len >= 4) {
   5722       zsm_swrite32( a, svNew );
   5723       a += 4;
   5724       len -= 4;
   5725       tl_assert(aligned64(a));
   5726    }
   5727    if (len == 0) return;
   5728 
   5729    if (len >= 8) {
   5730       tl_assert(aligned64(a));
   5731       while (len >= 8) {
   5732          zsm_swrite64( a, svNew );
   5733          a += 8;
   5734          len -= 8;
   5735       }
   5736       tl_assert(aligned64(a));
   5737    }
   5738    if (len == 0) return;
   5739 
   5740    if (len >= 4)
   5741       tl_assert(aligned32(a));
   5742    if (len >= 4) {
   5743       zsm_swrite32( a, svNew );
   5744       a += 4;
   5745       len -= 4;
   5746    }
   5747    if (len == 0) return;
   5748 
   5749    if (len >= 2)
   5750       tl_assert(aligned16(a));
   5751    if (len >= 2) {
   5752       zsm_swrite16( a, svNew );
   5753       a += 2;
   5754       len -= 2;
   5755    }
   5756    if (len == 0) return;
   5757 
   5758    if (len >= 1) {
   5759       zsm_swrite08( a, svNew );
   5760       //a += 1;
   5761       len -= 1;
   5762    }
   5763    tl_assert(len == 0);
   5764 }
   5765 
   5766 
   5767 /* If we're doing a small range, hand off to zsm_sset_range_SMALL.  But
   5768    for larger ranges, try to operate directly on the out-of-cache
   5769    representation, rather than dragging lines into the cache,
   5770    overwriting them, and forcing them out.  This turns out to be an
   5771    important performance optimisation.
   5772 
   5773    Note that this doesn't change the filtering arrangements.  The
   5774    caller of zsm_sset_range needs to attend to that. */
   5775 
   5776 static void zsm_sset_range ( Addr a, SizeT len, SVal svNew )
   5777 {
   5778    tl_assert(svNew != SVal_INVALID);
   5779    stats__cache_make_New_arange += (ULong)len;
   5780 
   5781    if (0 && len > 500)
   5782       VG_(printf)("make New      ( %#lx, %lu )\n", a, len );
   5783 
   5784    if (0) {
   5785       static UWord n_New_in_cache = 0;
   5786       static UWord n_New_not_in_cache = 0;
   5787       /* tag is 'a' with the in-line offset masked out,
   5788          eg a[31]..a[4] 0000 */
   5789       Addr       tag = a & ~(N_LINE_ARANGE - 1);
   5790       UWord      wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1);
   5791       if (LIKELY(tag == cache_shmem.tags0[wix])) {
   5792          n_New_in_cache++;
   5793       } else {
   5794          n_New_not_in_cache++;
   5795       }
   5796       if (0 == ((n_New_in_cache + n_New_not_in_cache) % 100000))
   5797          VG_(printf)("shadow_mem_make_New: IN %lu OUT %lu\n",
   5798                      n_New_in_cache, n_New_not_in_cache );
   5799    }
   5800 
   5801    if (LIKELY(len < 2 * N_LINE_ARANGE)) {
   5802       zsm_sset_range_SMALL( a, len, svNew );
   5803    } else {
   5804       Addr  before_start  = a;
   5805       Addr  aligned_start = cacheline_ROUNDUP(a);
   5806       Addr  after_start   = cacheline_ROUNDDN(a + len);
   5807       UWord before_len    = aligned_start - before_start;
   5808       UWord aligned_len   = after_start - aligned_start;
   5809       UWord after_len     = a + len - after_start;
   5810       tl_assert(before_start <= aligned_start);
   5811       tl_assert(aligned_start <= after_start);
   5812       tl_assert(before_len < N_LINE_ARANGE);
   5813       tl_assert(after_len < N_LINE_ARANGE);
   5814       tl_assert(get_cacheline_offset(aligned_start) == 0);
   5815       if (get_cacheline_offset(a) == 0) {
   5816          tl_assert(before_len == 0);
   5817          tl_assert(a == aligned_start);
   5818       }
   5819       if (get_cacheline_offset(a+len) == 0) {
   5820          tl_assert(after_len == 0);
   5821          tl_assert(after_start == a+len);
   5822       }
   5823       if (before_len > 0) {
   5824          zsm_sset_range_SMALL( before_start, before_len, svNew );
   5825       }
   5826       if (after_len > 0) {
   5827          zsm_sset_range_SMALL( after_start, after_len, svNew );
   5828       }
   5829       stats__cache_make_New_inZrep += (ULong)aligned_len;
   5830 
   5831       while (1) {
   5832          Addr tag;
   5833          UWord wix;
   5834          if (aligned_start >= after_start)
   5835             break;
   5836          tl_assert(get_cacheline_offset(aligned_start) == 0);
   5837          tag = aligned_start & ~(N_LINE_ARANGE - 1);
   5838          wix = (aligned_start >> N_LINE_BITS) & (N_WAY_NENT - 1);
   5839          if (tag == cache_shmem.tags0[wix]) {
   5840             UWord i;
   5841             for (i = 0; i < N_LINE_ARANGE / 8; i++)
   5842                zsm_swrite64( aligned_start + i * 8, svNew );
   5843          } else {
   5844             UWord i;
   5845             Word zix;
   5846             SecMap* sm;
   5847             LineZ* lineZ;
   5848             /* This line is not in the cache.  Do not force it in; instead
   5849                modify it in-place. */
   5850             /* find the Z line to write in and rcdec it or the
   5851                associated F line. */
   5852             find_Z_for_writing( &sm, &zix, tag );
   5853             tl_assert(sm);
   5854             tl_assert(zix >= 0 && zix < N_SECMAP_ZLINES);
   5855             lineZ = &sm->linesZ[zix];
   5856             lineZ->dict[0] = svNew;
   5857             lineZ->dict[1] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID;
   5858             for (i = 0; i < N_LINE_ARANGE/4; i++)
   5859                lineZ->ix2s[i] = 0; /* all refer to dict[0] */
   5860             rcinc_LineZ(lineZ);
   5861          }
   5862          aligned_start += N_LINE_ARANGE;
   5863          aligned_len -= N_LINE_ARANGE;
   5864       }
   5865       tl_assert(aligned_start == after_start);
   5866       tl_assert(aligned_len == 0);
   5867    }
   5868 }
   5869 
   5870 
   5871 /////////////////////////////////////////////////////////
   5872 //                                                     //
   5873 // Front-filtering accesses                            //
   5874 //                                                     //
   5875 /////////////////////////////////////////////////////////
   5876 
   5877 static UWord stats__f_ac = 0;
   5878 static UWord stats__f_sk = 0;
   5879 
   5880 #if 0
   5881 #  define STATS__F_SHOW \
   5882      do { \
   5883         if (UNLIKELY(0 == (stats__f_ac & 0xFFFFFF))) \
   5884            VG_(printf)("filters: ac %lu sk %lu\n",   \
   5885            stats__f_ac, stats__f_sk); \
   5886      } while (0)
   5887 #else
   5888 #  define STATS__F_SHOW /* */
   5889 #endif
   5890 
   5891 void zsm_sapply08_f__msmcwrite ( Thr* thr, Addr a ) {
   5892    stats__f_ac++;
   5893    STATS__F_SHOW;
   5894    if (LIKELY(Filter__ok_to_skip_cwr08(thr->filter, a))) {
   5895       stats__f_sk++;
   5896       return;
   5897    }
   5898    zsm_sapply08__msmcwrite(thr, a);
   5899 }
   5900 
   5901 void zsm_sapply16_f__msmcwrite ( Thr* thr, Addr a ) {
   5902    stats__f_ac++;
   5903    STATS__F_SHOW;
   5904    if (LIKELY(Filter__ok_to_skip_cwr16(thr->filter, a))) {
   5905       stats__f_sk++;
   5906       return;
   5907    }
   5908    zsm_sapply16__msmcwrite(thr, a);
   5909 }
   5910 
   5911 void zsm_sapply32_f__msmcwrite ( Thr* thr, Addr a ) {
   5912    stats__f_ac++;
   5913    STATS__F_SHOW;
   5914    if (LIKELY(Filter__ok_to_skip_cwr32(thr->filter, a))) {
   5915       stats__f_sk++;
   5916       return;
   5917    }
   5918    zsm_sapply32__msmcwrite(thr, a);
   5919 }
   5920 
   5921 void zsm_sapply64_f__msmcwrite ( Thr* thr, Addr a ) {
   5922    stats__f_ac++;
   5923    STATS__F_SHOW;
   5924    if (LIKELY(Filter__ok_to_skip_cwr64(thr->filter, a))) {
   5925       stats__f_sk++;
   5926       return;
   5927    }
   5928    zsm_sapply64__msmcwrite(thr, a);
   5929 }
   5930 
   5931 void zsm_sapplyNN_f__msmcwrite ( Thr* thr, Addr a, SizeT len )
   5932 {
   5933    /* fast track a couple of common cases */
   5934    if (len == 4 && aligned32(a)) {
   5935       zsm_sapply32_f__msmcwrite( thr, a );
   5936       return;
   5937    }
   5938    if (len == 8 && aligned64(a)) {
   5939       zsm_sapply64_f__msmcwrite( thr, a );
   5940       return;
   5941    }
   5942 
   5943    /* be completely general (but as efficient as possible) */
   5944    if (len == 0) return;
   5945 
   5946    if (!aligned16(a) && len >= 1) {
   5947       zsm_sapply08_f__msmcwrite( thr, a );
   5948       a += 1;
   5949       len -= 1;
   5950       tl_assert(aligned16(a));
   5951    }
   5952    if (len == 0) return;
   5953 
   5954    if (!aligned32(a) && len >= 2) {
   5955       zsm_sapply16_f__msmcwrite( thr, a );
   5956       a += 2;
   5957       len -= 2;
   5958       tl_assert(aligned32(a));
   5959    }
   5960    if (len == 0) return;
   5961 
   5962    if (!aligned64(a) && len >= 4) {
   5963       zsm_sapply32_f__msmcwrite( thr, a );
   5964       a += 4;
   5965       len -= 4;
   5966       tl_assert(aligned64(a));
   5967    }
   5968    if (len == 0) return;
   5969 
   5970    if (len >= 8) {
   5971       tl_assert(aligned64(a));
   5972       while (len >= 8) {
   5973          zsm_sapply64_f__msmcwrite( thr, a );
   5974          a += 8;
   5975          len -= 8;
   5976       }
   5977       tl_assert(aligned64(a));
   5978    }
   5979    if (len == 0) return;
   5980 
   5981    if (len >= 4)
   5982       tl_assert(aligned32(a));
   5983    if (len >= 4) {
   5984       zsm_sapply32_f__msmcwrite( thr, a );
   5985       a += 4;
   5986       len -= 4;
   5987    }
   5988    if (len == 0) return;
   5989 
   5990    if (len >= 2)
   5991       tl_assert(aligned16(a));
   5992    if (len >= 2) {
   5993       zsm_sapply16_f__msmcwrite( thr, a );
   5994       a += 2;
   5995       len -= 2;
   5996    }
   5997    if (len == 0) return;
   5998 
   5999    if (len >= 1) {
   6000       zsm_sapply08_f__msmcwrite( thr, a );
   6001       //a += 1;
   6002       len -= 1;
   6003    }
   6004    tl_assert(len == 0);
   6005 }
   6006 
   6007 void zsm_sapply08_f__msmcread ( Thr* thr, Addr a ) {
   6008    stats__f_ac++;
   6009    STATS__F_SHOW;
   6010    if (LIKELY(Filter__ok_to_skip_crd08(thr->filter, a))) {
   6011       stats__f_sk++;
   6012       return;
   6013    }
   6014    zsm_sapply08__msmcread(thr, a);
   6015 }
   6016 
   6017 void zsm_sapply16_f__msmcread ( Thr* thr, Addr a ) {
   6018    stats__f_ac++;
   6019    STATS__F_SHOW;
   6020    if (LIKELY(Filter__ok_to_skip_crd16(thr->filter, a))) {
   6021       stats__f_sk++;
   6022       return;
   6023    }
   6024    zsm_sapply16__msmcread(thr, a);
   6025 }
   6026 
   6027 void zsm_sapply32_f__msmcread ( Thr* thr, Addr a ) {
   6028    stats__f_ac++;
   6029    STATS__F_SHOW;
   6030    if (LIKELY(Filter__ok_to_skip_crd32(thr->filter, a))) {
   6031       stats__f_sk++;
   6032       return;
   6033    }
   6034    zsm_sapply32__msmcread(thr, a);
   6035 }
   6036 
   6037 void zsm_sapply64_f__msmcread ( Thr* thr, Addr a ) {
   6038    stats__f_ac++;
   6039    STATS__F_SHOW;
   6040    if (LIKELY(Filter__ok_to_skip_crd64(thr->filter, a))) {
   6041       stats__f_sk++;
   6042       return;
   6043    }
   6044    zsm_sapply64__msmcread(thr, a);
   6045 }
   6046 
   6047 void zsm_sapplyNN_f__msmcread ( Thr* thr, Addr a, SizeT len )
   6048 {
   6049    /* fast track a couple of common cases */
   6050    if (len == 4 && aligned32(a)) {
   6051       zsm_sapply32_f__msmcread( thr, a );
   6052       return;
   6053    }
   6054    if (len == 8 && aligned64(a)) {
   6055       zsm_sapply64_f__msmcread( thr, a );
   6056       return;
   6057    }
   6058 
   6059    /* be completely general (but as efficient as possible) */
   6060    if (len == 0) return;
   6061 
   6062    if (!aligned16(a) && len >= 1) {
   6063       zsm_sapply08_f__msmcread( thr, a );
   6064       a += 1;
   6065       len -= 1;
   6066       tl_assert(aligned16(a));
   6067    }
   6068    if (len == 0) return;
   6069 
   6070    if (!aligned32(a) && len >= 2) {
   6071       zsm_sapply16_f__msmcread( thr, a );
   6072       a += 2;
   6073       len -= 2;
   6074       tl_assert(aligned32(a));
   6075    }
   6076    if (len == 0) return;
   6077 
   6078    if (!aligned64(a) && len >= 4) {
   6079       zsm_sapply32_f__msmcread( thr, a );
   6080       a += 4;
   6081       len -= 4;
   6082       tl_assert(aligned64(a));
   6083    }
   6084    if (len == 0) return;
   6085 
   6086    if (len >= 8) {
   6087       tl_assert(aligned64(a));
   6088       while (len >= 8) {
   6089          zsm_sapply64_f__msmcread( thr, a );
   6090          a += 8;
   6091          len -= 8;
   6092       }
   6093       tl_assert(aligned64(a));
   6094    }
   6095    if (len == 0) return;
   6096 
   6097    if (len >= 4)
   6098       tl_assert(aligned32(a));
   6099    if (len >= 4) {
   6100       zsm_sapply32_f__msmcread( thr, a );
   6101       a += 4;
   6102       len -= 4;
   6103    }
   6104    if (len == 0) return;
   6105 
   6106    if (len >= 2)
   6107       tl_assert(aligned16(a));
   6108    if (len >= 2) {
   6109       zsm_sapply16_f__msmcread( thr, a );
   6110       a += 2;
   6111       len -= 2;
   6112    }
   6113    if (len == 0) return;
   6114 
   6115    if (len >= 1) {
   6116       zsm_sapply08_f__msmcread( thr, a );
   6117       //a += 1;
   6118       len -= 1;
   6119    }
   6120    tl_assert(len == 0);
   6121 }
   6122 
   6123 void libhb_Thr_resumes ( Thr* thr )
   6124 {
   6125    if (0) VG_(printf)("resume %p\n", thr);
   6126    tl_assert(thr);
   6127    tl_assert(!thr->llexit_done);
   6128    Filter__clear(thr->filter, "libhb_Thr_resumes");
   6129    /* A kludge, but .. if this thread doesn't have any marker stacks
   6130       at all, get one right now.  This is easier than figuring out
   6131       exactly when at thread startup we can and can't take a stack
   6132       snapshot. */
   6133    if (HG_(clo_history_level) == 1) {
   6134       tl_assert(thr->local_Kws_n_stacks);
   6135       if (VG_(sizeXA)( thr->local_Kws_n_stacks ) == 0)
   6136          note_local_Kw_n_stack_for(thr);
   6137    }
   6138 }
   6139 
   6140 
   6141 /////////////////////////////////////////////////////////
   6142 //                                                     //
   6143 // Synchronisation objects                             //
   6144 //                                                     //
   6145 /////////////////////////////////////////////////////////
   6146 
   6147 /* A double linked list of all the SO's. */
   6148 SO* admin_SO = NULL;
   6149 
   6150 static SO* SO__Alloc ( void )
   6151 {
   6152    SO* so = HG_(zalloc)( "libhb.SO__Alloc.1", sizeof(SO) );
   6153    so->viR   = VtsID_INVALID;
   6154    so->viW   = VtsID_INVALID;
   6155    so->magic = SO_MAGIC;
   6156    /* Add to double linked list */
   6157    if (admin_SO) {
   6158       tl_assert(admin_SO->admin_prev == NULL);
   6159       admin_SO->admin_prev = so;
   6160       so->admin_next = admin_SO;
   6161    } else {
   6162       so->admin_next = NULL;
   6163    }
   6164    so->admin_prev = NULL;
   6165    admin_SO = so;
   6166    /* */
   6167    return so;
   6168 }
   6169 
   6170 static void SO__Dealloc ( SO* so )
   6171 {
   6172    tl_assert(so);
   6173    tl_assert(so->magic == SO_MAGIC);
   6174    if (so->viR == VtsID_INVALID) {
   6175       tl_assert(so->viW == VtsID_INVALID);
   6176    } else {
   6177       tl_assert(so->viW != VtsID_INVALID);
   6178       VtsID__rcdec(so->viR);
   6179       VtsID__rcdec(so->viW);
   6180    }
   6181    so->magic = 0;
   6182    /* Del from double linked list */
   6183    if (so->admin_prev)
   6184       so->admin_prev->admin_next = so->admin_next;
   6185    if (so->admin_next)
   6186       so->admin_next->admin_prev = so->admin_prev;
   6187    if (so == admin_SO)
   6188       admin_SO = so->admin_next;
   6189    /* */
   6190    HG_(free)( so );
   6191 }
   6192 
   6193 
   6194 /////////////////////////////////////////////////////////
   6195 //                                                     //
   6196 // Top Level API                                       //
   6197 //                                                     //
   6198 /////////////////////////////////////////////////////////
   6199 
   6200 static void show_thread_state ( const HChar* str, Thr* t )
   6201 {
   6202    if (1) return;
   6203    if (t->viR == t->viW) {
   6204       VG_(printf)("thr \"%s\" %p has vi* %u==", str, t, t->viR );
   6205       VtsID__pp( t->viR );
   6206       VG_(printf)("%s","\n");
   6207    } else {
   6208       VG_(printf)("thr \"%s\" %p has viR %u==", str, t, t->viR );
   6209       VtsID__pp( t->viR );
   6210       VG_(printf)(" viW %u==", t->viW);
   6211       VtsID__pp( t->viW );
   6212       VG_(printf)("%s","\n");
   6213    }
   6214 }
   6215 
   6216 
   6217 Thr* libhb_init (
   6218         void        (*get_stacktrace)( Thr*, Addr*, UWord ),
   6219         ExeContext* (*get_EC)( Thr* )
   6220      )
   6221 {
   6222    Thr*  thr;
   6223    VtsID vi;
   6224 
   6225    // We will have to have to store a large number of these,
   6226    // so make sure they're the size we expect them to be.
   6227    STATIC_ASSERT(sizeof(ScalarTS) == 8);
   6228 
   6229    /* because first 1024 unusable */
   6230    STATIC_ASSERT(SCALARTS_N_THRBITS >= 11);
   6231    /* so as to fit in a UInt w/ 5 bits to spare (see defn of
   6232       Thr_n_RCEC and TSW). */
   6233    STATIC_ASSERT(SCALARTS_N_THRBITS <= 27);
   6234 
   6235    /* Need to be sure that Thr_n_RCEC is 2 words (64-bit) or 3 words
   6236       (32-bit).  It's not correctness-critical, but there are a lot of
   6237       them, so it's important from a space viewpoint.  Unfortunately
   6238       we simply can't pack it into 2 words on a 32-bit target. */
   6239    STATIC_ASSERT(   (sizeof(UWord) == 8 && sizeof(Thr_n_RCEC) == 16)
   6240                  || (sizeof(UWord) == 4 && sizeof(Thr_n_RCEC) == 12));
   6241    STATIC_ASSERT(sizeof(TSW) == sizeof(UInt));
   6242 
   6243    /* Word sets really are 32 bits.  Even on a 64 bit target. */
   6244    STATIC_ASSERT(sizeof(WordSetID) == 4);
   6245    STATIC_ASSERT(sizeof(WordSet) == sizeof(WordSetID));
   6246 
   6247    tl_assert(get_stacktrace);
   6248    tl_assert(get_EC);
   6249    main_get_stacktrace   = get_stacktrace;
   6250    main_get_EC           = get_EC;
   6251 
   6252    // No need to initialise hg_wordfm.
   6253    // No need to initialise hg_wordset.
   6254 
   6255    /* Allocated once and never deallocated.  Used as a temporary in
   6256       VTS singleton, tick and join operations. */
   6257    temp_max_sized_VTS = VTS__new( "libhb.libhb_init.1", ThrID_MAX_VALID );
   6258    temp_max_sized_VTS->id = VtsID_INVALID;
   6259    verydead_thread_tables_init();
   6260    vts_set_init();
   6261    vts_tab_init();
   6262    event_map_init();
   6263    VtsID__invalidate_caches();
   6264 
   6265    // initialise shadow memory
   6266    zsm_init( );
   6267 
   6268    thr = Thr__new();
   6269    vi  = VtsID__mk_Singleton( thr, 1 );
   6270    thr->viR = vi;
   6271    thr->viW = vi;
   6272    VtsID__rcinc(thr->viR);
   6273    VtsID__rcinc(thr->viW);
   6274 
   6275    show_thread_state("  root", thr);
   6276    return thr;
   6277 }
   6278 
   6279 
   6280 Thr* libhb_create ( Thr* parent )
   6281 {
   6282    /* The child's VTSs are copies of the parent's VTSs, but ticked at
   6283       the child's index.  Since the child's index is guaranteed
   6284       unique, it has never been seen before, so the implicit value
   6285       before the tick is zero and after that is one. */
   6286    Thr* child = Thr__new();
   6287 
   6288    child->viR = VtsID__tick( parent->viR, child );
   6289    child->viW = VtsID__tick( parent->viW, child );
   6290    Filter__clear(child->filter, "libhb_create(child)");
   6291    VtsID__rcinc(child->viR);
   6292    VtsID__rcinc(child->viW);
   6293    /* We need to do note_local_Kw_n_stack_for( child ), but it's too
   6294       early for that - it may not have a valid TId yet.  So, let
   6295       libhb_Thr_resumes pick it up the first time the thread runs. */
   6296 
   6297    tl_assert(VtsID__indexAt( child->viR, child ) == 1);
   6298    tl_assert(VtsID__indexAt( child->viW, child ) == 1);
   6299 
   6300    /* and the parent has to move along too */
   6301    VtsID__rcdec(parent->viR);
   6302    VtsID__rcdec(parent->viW);
   6303    parent->viR = VtsID__tick( parent->viR, parent );
   6304    parent->viW = VtsID__tick( parent->viW, parent );
   6305    Filter__clear(parent->filter, "libhb_create(parent)");
   6306    VtsID__rcinc(parent->viR);
   6307    VtsID__rcinc(parent->viW);
   6308    note_local_Kw_n_stack_for( parent );
   6309 
   6310    show_thread_state(" child", child);
   6311    show_thread_state("parent", parent);
   6312 
   6313    return child;
   6314 }
   6315 
   6316 /* Shut down the library, and print stats (in fact that's _all_
   6317    this is for. */
   6318 void libhb_shutdown ( Bool show_stats )
   6319 {
   6320    if (show_stats) {
   6321       VG_(printf)("%s","<<< BEGIN libhb stats >>>\n");
   6322       VG_(printf)(" secmaps: %'10lu allocd (%'12lu g-a-range)\n",
   6323                   stats__secmaps_allocd,
   6324                   stats__secmap_ga_space_covered);
   6325       VG_(printf)("  linesZ: %'10lu allocd (%'12lu bytes occupied)\n",
   6326                   stats__secmap_linesZ_allocd,
   6327                   stats__secmap_linesZ_bytes);
   6328       VG_(printf)("  linesF: %'10lu allocd (%'12lu bytes occupied)"
   6329                   " (%'10lu used)\n",
   6330                   VG_(sizePA) (LineF_pool_allocator),
   6331                   VG_(sizePA) (LineF_pool_allocator) * sizeof(LineF),
   6332                   shmem__SecMap_used_linesF());
   6333       VG_(printf)(" secmaps: %'10lu in map (can be scanGCed %'5lu)"
   6334                   " #%lu scanGC \n",
   6335                   stats__secmaps_in_map_shmem,
   6336                   shmem__SecMap_do_GC(False /* really do GC */),
   6337                   stats__secmaps_scanGC);
   6338       tl_assert (VG_(sizeFM) (map_shmem) == stats__secmaps_in_map_shmem);
   6339       VG_(printf)(" secmaps: %'10lu in freelist,"
   6340                   " total (scanGCed %'lu, ssetGCed %'lu)\n",
   6341                   SecMap_freelist_length(),
   6342                   stats__secmaps_scanGCed,
   6343                   stats__secmaps_ssetGCed);
   6344       VG_(printf)(" secmaps: %'10lu searches (%'12lu slow)\n",
   6345                   stats__secmaps_search, stats__secmaps_search_slow);
   6346 
   6347       VG_(printf)("%s","\n");
   6348       VG_(printf)("   cache: %'lu totrefs (%'lu misses)\n",
   6349                   stats__cache_totrefs, stats__cache_totmisses );
   6350       VG_(printf)("   cache: %'14lu Z-fetch,    %'14lu F-fetch\n",
   6351                   stats__cache_Z_fetches, stats__cache_F_fetches );
   6352       VG_(printf)("   cache: %'14lu Z-wback,    %'14lu F-wback\n",
   6353                   stats__cache_Z_wbacks, stats__cache_F_wbacks );
   6354       VG_(printf)("   cache: %'14lu flushes_invals\n",
   6355                   stats__cache_flushes_invals );
   6356       VG_(printf)("   cache: %'14llu arange_New  %'14llu direct-to-Zreps\n",
   6357                   stats__cache_make_New_arange,
   6358                   stats__cache_make_New_inZrep);
   6359 
   6360       VG_(printf)("%s","\n");
   6361       VG_(printf)("   cline: %'10lu normalises\n",
   6362                   stats__cline_normalises );
   6363       VG_(printf)("   cline: c rds 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n",
   6364                   stats__cline_cread64s,
   6365                   stats__cline_cread32s,
   6366                   stats__cline_cread16s,
   6367                   stats__cline_cread08s );
   6368       VG_(printf)("   cline: c wrs 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n",
   6369                   stats__cline_cwrite64s,
   6370                   stats__cline_cwrite32s,
   6371                   stats__cline_cwrite16s,
   6372                   stats__cline_cwrite08s );
   6373       VG_(printf)("   cline: s wrs 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n",
   6374                   stats__cline_swrite64s,
   6375                   stats__cline_swrite32s,
   6376                   stats__cline_swrite16s,
   6377                   stats__cline_swrite08s );
   6378       VG_(printf)("   cline: s rd1s %'lu, s copy1s %'lu\n",
   6379                   stats__cline_sread08s, stats__cline_scopy08s );
   6380       VG_(printf)("   cline:    splits: 8to4 %'12lu    4to2 %'12lu"
   6381                   "    2to1 %'12lu\n",
   6382                   stats__cline_64to32splits, stats__cline_32to16splits,
   6383                   stats__cline_16to8splits );
   6384       VG_(printf)("   cline: pulldowns: 8to4 %'12lu    4to2 %'12lu"
   6385                   "    2to1 %'12lu\n",
   6386                   stats__cline_64to32pulldown, stats__cline_32to16pulldown,
   6387                   stats__cline_16to8pulldown );
   6388       if (0)
   6389       VG_(printf)("   cline: sizeof(CacheLineZ) %ld,"
   6390                   " covers %ld bytes of arange\n",
   6391                   (Word)sizeof(LineZ),
   6392                   (Word)N_LINE_ARANGE);
   6393 
   6394       VG_(printf)("%s","\n");
   6395 
   6396       VG_(printf)("   libhb: %'13llu msmcread  (%'llu dragovers)\n",
   6397                   stats__msmcread, stats__msmcread_change);
   6398       VG_(printf)("   libhb: %'13llu msmcwrite (%'llu dragovers)\n",
   6399                   stats__msmcwrite, stats__msmcwrite_change);
   6400       VG_(printf)("   libhb: %'13llu cmpLEQ queries (%'llu misses)\n",
   6401                   stats__cmpLEQ_queries, stats__cmpLEQ_misses);
   6402       VG_(printf)("   libhb: %'13llu join2  queries (%'llu misses)\n",
   6403                   stats__join2_queries, stats__join2_misses);
   6404 
   6405       VG_(printf)("%s","\n");
   6406       VG_(printf)("   libhb: VTSops: tick %'lu,  join %'lu,  cmpLEQ %'lu\n",
   6407                   stats__vts__tick, stats__vts__join,  stats__vts__cmpLEQ );
   6408       VG_(printf)("   libhb: VTSops: cmp_structural %'lu (%'lu slow)\n",
   6409                   stats__vts__cmp_structural, stats__vts__cmp_structural_slow);
   6410       VG_(printf)("   libhb: VTSset: find__or__clone_and_add %'lu"
   6411                   " (%'lu allocd)\n",
   6412                    stats__vts_set__focaa, stats__vts_set__focaa_a );
   6413       VG_(printf)( "   libhb: VTSops: indexAt_SLOW %'lu\n",
   6414                    stats__vts__indexat_slow );
   6415 
   6416       VG_(printf)("%s","\n");
   6417       VG_(printf)(
   6418          "   libhb: %ld entries in vts_table (approximately %lu bytes)\n",
   6419          VG_(sizeXA)( vts_tab ), VG_(sizeXA)( vts_tab ) * sizeof(VtsTE)
   6420       );
   6421       VG_(printf)("   libhb: #%lu vts_tab GC    #%lu vts pruning\n",
   6422                   stats__vts_tab_GC, stats__vts_pruning);
   6423       VG_(printf)( "   libhb: %lu entries in vts_set\n",
   6424                    VG_(sizeFM)( vts_set ) );
   6425 
   6426       VG_(printf)("%s","\n");
   6427       {
   6428          UInt live = 0;
   6429          UInt llexit_done = 0;
   6430          UInt joinedwith_done = 0;
   6431          UInt llexit_and_joinedwith_done = 0;
   6432 
   6433          Thread* hgthread = get_admin_threads();
   6434          tl_assert(hgthread);
   6435          while (hgthread) {
   6436             Thr* hbthr = hgthread->hbthr;
   6437             tl_assert(hbthr);
   6438             if (hbthr->llexit_done && hbthr->joinedwith_done)
   6439                llexit_and_joinedwith_done++;
   6440             else if (hbthr->llexit_done)
   6441                llexit_done++;
   6442             else if (hbthr->joinedwith_done)
   6443                joinedwith_done++;
   6444             else
   6445                live++;
   6446             hgthread = hgthread->admin;
   6447          }
   6448          VG_(printf)("   libhb: threads live: %u exit_and_joinedwith %u"
   6449                      " exit %u joinedwith %u\n",
   6450                      live, llexit_and_joinedwith_done,
   6451                      llexit_done, joinedwith_done);
   6452          VG_(printf)("   libhb: %d verydead_threads, "
   6453                      "%d verydead_threads_not_pruned\n",
   6454                      (int) VG_(sizeXA)( verydead_thread_table),
   6455                      (int) VG_(sizeXA)( verydead_thread_table_not_pruned));
   6456          tl_assert (VG_(sizeXA)( verydead_thread_table)
   6457                     + VG_(sizeXA)( verydead_thread_table_not_pruned)
   6458                     == llexit_and_joinedwith_done);
   6459       }
   6460 
   6461       VG_(printf)("%s","\n");
   6462       VG_(printf)( "   libhb: oldrefHTN %lu (%'d bytes)\n",
   6463                    oldrefHTN, (int)(oldrefHTN * sizeof(OldRef)));
   6464       tl_assert (oldrefHTN == VG_(HT_count_nodes) (oldrefHT));
   6465       VG_(printf)( "   libhb: oldref lookup found=%lu notfound=%lu\n",
   6466                    stats__evm__lookup_found, stats__evm__lookup_notfound);
   6467       if (VG_(clo_verbosity) > 1)
   6468          VG_(HT_print_stats) (oldrefHT, cmp_oldref_tsw);
   6469       VG_(printf)( "   libhb: oldref bind tsw/rcec "
   6470                    "==/==:%'lu ==/!=:%'lu !=/!=:%'lu\n",
   6471                    stats__ctxt_eq_tsw_eq_rcec, stats__ctxt_eq_tsw_neq_rcec,
   6472                    stats__ctxt_neq_tsw_neq_rcec);
   6473       VG_(printf)( "   libhb: ctxt__rcdec calls %'lu. rcec gc discards %'lu\n",
   6474                    stats__ctxt_rcdec_calls, stats__ctxt_rcec_gc_discards);
   6475       VG_(printf)( "   libhb: contextTab: %lu slots,"
   6476                    " %lu cur ents(ref'd %lu),"
   6477                    " %lu max ents\n",
   6478                    (UWord)N_RCEC_TAB,
   6479                    stats__ctxt_tab_curr, RCEC_referenced,
   6480                    stats__ctxt_tab_max );
   6481       {
   6482 #        define  MAXCHAIN 10
   6483          UInt chains[MAXCHAIN+1]; // [MAXCHAIN] gets all chains >= MAXCHAIN
   6484          UInt non0chain = 0;
   6485          UInt n;
   6486          UInt i;
   6487          RCEC *p;
   6488 
   6489          for (i = 0; i <= MAXCHAIN; i++) chains[i] = 0;
   6490          for (i = 0; i < N_RCEC_TAB; i++) {
   6491             n = 0;
   6492             for (p = contextTab[i]; p; p = p->next)
   6493                n++;
   6494             if (n < MAXCHAIN)
   6495                chains[n]++;
   6496             else
   6497                chains[MAXCHAIN]++;
   6498             if (n > 0)
   6499                non0chain++;
   6500          }
   6501          VG_(printf)( "   libhb: contextTab chain of [length]=nchain."
   6502                       " Avg chain len %3.1f\n"
   6503                       "        ",
   6504                       (Double)stats__ctxt_tab_curr
   6505                       / (Double)(non0chain ? non0chain : 1));
   6506          for (i = 0; i <= MAXCHAIN; i++) {
   6507             if (chains[i] != 0)
   6508                 VG_(printf)( "[%u%s]=%u ",
   6509                              i, i == MAXCHAIN ? "+" : "",
   6510                              chains[i]);
   6511          }
   6512          VG_(printf)( "\n");
   6513 #        undef MAXCHAIN
   6514       }
   6515       VG_(printf)( "   libhb: contextTab: %lu queries, %lu cmps\n",
   6516                    stats__ctxt_tab_qs,
   6517                    stats__ctxt_tab_cmps );
   6518 #if 0
   6519       VG_(printf)("sizeof(AvlNode)     = %lu\n", sizeof(AvlNode));
   6520       VG_(printf)("sizeof(WordBag)     = %lu\n", sizeof(WordBag));
   6521       VG_(printf)("sizeof(MaybeWord)   = %lu\n", sizeof(MaybeWord));
   6522       VG_(printf)("sizeof(CacheLine)   = %lu\n", sizeof(CacheLine));
   6523       VG_(printf)("sizeof(LineZ)       = %lu\n", sizeof(LineZ));
   6524       VG_(printf)("sizeof(LineF)       = %lu\n", sizeof(LineF));
   6525       VG_(printf)("sizeof(SecMap)      = %lu\n", sizeof(SecMap));
   6526       VG_(printf)("sizeof(Cache)       = %lu\n", sizeof(Cache));
   6527       VG_(printf)("sizeof(SMCacheEnt)  = %lu\n", sizeof(SMCacheEnt));
   6528       VG_(printf)("sizeof(CountedSVal) = %lu\n", sizeof(CountedSVal));
   6529       VG_(printf)("sizeof(VTS)         = %lu\n", sizeof(VTS));
   6530       VG_(printf)("sizeof(ScalarTS)    = %lu\n", sizeof(ScalarTS));
   6531       VG_(printf)("sizeof(VtsTE)       = %lu\n", sizeof(VtsTE));
   6532       VG_(printf)("sizeof(MSMInfo)     = %lu\n", sizeof(MSMInfo));
   6533 
   6534       VG_(printf)("sizeof(struct _XArray)     = %lu\n", sizeof(struct _XArray));
   6535       VG_(printf)("sizeof(struct _WordFM)     = %lu\n", sizeof(struct _WordFM));
   6536       VG_(printf)("sizeof(struct _Thr)     = %lu\n", sizeof(struct _Thr));
   6537       VG_(printf)("sizeof(struct _SO)     = %lu\n", sizeof(struct _SO));
   6538 #endif
   6539 
   6540       VG_(printf)("%s","<<< END libhb stats >>>\n");
   6541       VG_(printf)("%s","\n");
   6542 
   6543    }
   6544 }
   6545 
   6546 /* Receive notification that a thread has low level exited.  The
   6547    significance here is that we do not expect to see any more memory
   6548    references from it. */
   6549 void libhb_async_exit ( Thr* thr )
   6550 {
   6551    tl_assert(thr);
   6552    tl_assert(!thr->llexit_done);
   6553    thr->llexit_done = True;
   6554 
   6555    /* free up Filter and local_Kws_n_stacks (well, actually not the
   6556       latter ..) */
   6557    tl_assert(thr->filter);
   6558    HG_(free)(thr->filter);
   6559    thr->filter = NULL;
   6560 
   6561    /* Tell the VTS mechanism this thread has exited, so it can
   6562       participate in VTS pruning.  Note this can only happen if the
   6563       thread has both ll_exited and has been joined with. */
   6564    if (thr->joinedwith_done)
   6565       VTS__declare_thread_very_dead(thr);
   6566 
   6567    /* Another space-accuracy tradeoff.  Do we want to be able to show
   6568       H1 history for conflicts in threads which have since exited?  If
   6569       yes, then we better not free up thr->local_Kws_n_stacks.  The
   6570       downside is a potential per-thread leak of up to
   6571       N_KWs_N_STACKs_PER_THREAD * sizeof(ULong_n_EC) * whatever the
   6572       XArray average overcommit factor is (1.5 I'd guess). */
   6573    // hence:
   6574    // VG_(deleteXA)(thr->local_Kws_n_stacks);
   6575    // thr->local_Kws_n_stacks = NULL;
   6576 }
   6577 
   6578 /* Receive notification that a thread has been joined with.  The
   6579    significance here is that we do not expect to see any further
   6580    references to its vector clocks (Thr::viR and Thr::viW). */
   6581 void libhb_joinedwith_done ( Thr* thr )
   6582 {
   6583    tl_assert(thr);
   6584    /* Caller must ensure that this is only ever called once per Thr. */
   6585    tl_assert(!thr->joinedwith_done);
   6586    thr->joinedwith_done = True;
   6587    if (thr->llexit_done)
   6588       VTS__declare_thread_very_dead(thr);
   6589 }
   6590 
   6591 
   6592 /* Both Segs and SOs point to VTSs.  However, there is no sharing, so
   6593    a Seg that points at a VTS is its one-and-only owner, and ditto for
   6594    a SO that points at a VTS. */
   6595 
   6596 SO* libhb_so_alloc ( void )
   6597 {
   6598    return SO__Alloc();
   6599 }
   6600 
   6601 void libhb_so_dealloc ( SO* so )
   6602 {
   6603    tl_assert(so);
   6604    tl_assert(so->magic == SO_MAGIC);
   6605    SO__Dealloc(so);
   6606 }
   6607 
   6608 /* See comments in libhb.h for details on the meaning of
   6609    strong vs weak sends and strong vs weak receives. */
   6610 void libhb_so_send ( Thr* thr, SO* so, Bool strong_send )
   6611 {
   6612    /* Copy the VTSs from 'thr' into the sync object, and then move
   6613       the thread along one step. */
   6614 
   6615    tl_assert(so);
   6616    tl_assert(so->magic == SO_MAGIC);
   6617 
   6618    /* stay sane .. a thread's read-clock must always lead or be the
   6619       same as its write-clock */
   6620    { Bool leq = VtsID__cmpLEQ(thr->viW, thr->viR);
   6621      tl_assert(leq);
   6622    }
   6623 
   6624    /* since we're overwriting the VtsIDs in the SO, we need to drop
   6625       any references made by the previous contents thereof */
   6626    if (so->viR == VtsID_INVALID) {
   6627       tl_assert(so->viW == VtsID_INVALID);
   6628       so->viR = thr->viR;
   6629       so->viW = thr->viW;
   6630       VtsID__rcinc(so->viR);
   6631       VtsID__rcinc(so->viW);
   6632    } else {
   6633       /* In a strong send, we dump any previous VC in the SO and
   6634          install the sending thread's VC instead.  For a weak send we
   6635          must join2 with what's already there. */
   6636       tl_assert(so->viW != VtsID_INVALID);
   6637       VtsID__rcdec(so->viR);
   6638       VtsID__rcdec(so->viW);
   6639       so->viR = strong_send ? thr->viR : VtsID__join2( so->viR, thr->viR );
   6640       so->viW = strong_send ? thr->viW : VtsID__join2( so->viW, thr->viW );
   6641       VtsID__rcinc(so->viR);
   6642       VtsID__rcinc(so->viW);
   6643    }
   6644 
   6645    /* move both parent clocks along */
   6646    VtsID__rcdec(thr->viR);
   6647    VtsID__rcdec(thr->viW);
   6648    thr->viR = VtsID__tick( thr->viR, thr );
   6649    thr->viW = VtsID__tick( thr->viW, thr );
   6650    if (!thr->llexit_done) {
   6651       Filter__clear(thr->filter, "libhb_so_send");
   6652       note_local_Kw_n_stack_for(thr);
   6653    }
   6654    VtsID__rcinc(thr->viR);
   6655    VtsID__rcinc(thr->viW);
   6656 
   6657    if (strong_send)
   6658       show_thread_state("s-send", thr);
   6659    else
   6660       show_thread_state("w-send", thr);
   6661 }
   6662 
   6663 void libhb_so_recv ( Thr* thr, SO* so, Bool strong_recv )
   6664 {
   6665    tl_assert(so);
   6666    tl_assert(so->magic == SO_MAGIC);
   6667 
   6668    if (so->viR != VtsID_INVALID) {
   6669       tl_assert(so->viW != VtsID_INVALID);
   6670 
   6671       /* Weak receive (basically, an R-acquisition of a R-W lock).
   6672          This advances the read-clock of the receiver, but not the
   6673          write-clock. */
   6674       VtsID__rcdec(thr->viR);
   6675       thr->viR = VtsID__join2( thr->viR, so->viR );
   6676       VtsID__rcinc(thr->viR);
   6677 
   6678       /* At one point (r10589) it seemed safest to tick the clocks for
   6679          the receiving thread after the join.  But on reflection, I
   6680          wonder if that might cause it to 'overtake' constraints,
   6681          which could lead to missing races.  So, back out that part of
   6682          r10589. */
   6683       //VtsID__rcdec(thr->viR);
   6684       //thr->viR = VtsID__tick( thr->viR, thr );
   6685       //VtsID__rcinc(thr->viR);
   6686 
   6687       /* For a strong receive, we also advance the receiver's write
   6688          clock, which means the receive as a whole is essentially
   6689          equivalent to a W-acquisition of a R-W lock. */
   6690       if (strong_recv) {
   6691          VtsID__rcdec(thr->viW);
   6692          thr->viW = VtsID__join2( thr->viW, so->viW );
   6693          VtsID__rcinc(thr->viW);
   6694 
   6695          /* See comment just above, re r10589. */
   6696          //VtsID__rcdec(thr->viW);
   6697          //thr->viW = VtsID__tick( thr->viW, thr );
   6698          //VtsID__rcinc(thr->viW);
   6699       }
   6700 
   6701       if (thr->filter)
   6702          Filter__clear(thr->filter, "libhb_so_recv");
   6703       note_local_Kw_n_stack_for(thr);
   6704 
   6705       if (strong_recv)
   6706          show_thread_state("s-recv", thr);
   6707       else
   6708          show_thread_state("w-recv", thr);
   6709 
   6710    } else {
   6711       tl_assert(so->viW == VtsID_INVALID);
   6712       /* Deal with degenerate case: 'so' has no vts, so there has been
   6713          no message posted to it.  Just ignore this case. */
   6714       show_thread_state("d-recv", thr);
   6715    }
   6716 }
   6717 
   6718 Bool libhb_so_everSent ( SO* so )
   6719 {
   6720    if (so->viR == VtsID_INVALID) {
   6721       tl_assert(so->viW == VtsID_INVALID);
   6722       return False;
   6723    } else {
   6724       tl_assert(so->viW != VtsID_INVALID);
   6725       return True;
   6726    }
   6727 }
   6728 
   6729 #define XXX1 0 // 0x67a106c
   6730 #define XXX2 0
   6731 
   6732 static inline Bool TRACEME(Addr a, SizeT szB) {
   6733    if (XXX1 && a <= XXX1 && XXX1 <= a+szB) return True;
   6734    if (XXX2 && a <= XXX2 && XXX2 <= a+szB) return True;
   6735    return False;
   6736 }
   6737 static void trace ( Thr* thr, Addr a, SizeT szB, const HChar* s )
   6738 {
   6739   SVal sv = zsm_sread08(a);
   6740   VG_(printf)("thr %p (%#lx,%lu) %s: 0x%016llx ", thr,a,szB,s,sv);
   6741   show_thread_state("", thr);
   6742   VG_(printf)("%s","\n");
   6743 }
   6744 
   6745 void libhb_srange_new ( Thr* thr, Addr a, SizeT szB )
   6746 {
   6747    SVal sv = SVal__mkC(thr->viW, thr->viW);
   6748    tl_assert(is_sane_SVal_C(sv));
   6749    if (0 && TRACEME(a,szB)) trace(thr,a,szB,"nw-before");
   6750    zsm_sset_range( a, szB, sv );
   6751    Filter__clear_range( thr->filter, a, szB );
   6752    if (0 && TRACEME(a,szB)) trace(thr,a,szB,"nw-after ");
   6753 }
   6754 
   6755 void libhb_srange_noaccess_NoFX ( Thr* thr, Addr a, SizeT szB )
   6756 {
   6757    /* do nothing */
   6758 }
   6759 
   6760 
   6761 /* Set the lines zix_start till zix_end to NOACCESS. */
   6762 static void zsm_secmap_line_range_noaccess (SecMap *sm,
   6763                                             UInt zix_start, UInt zix_end)
   6764 {
   6765    for (UInt lz = zix_start; lz <= zix_end; lz++) {
   6766       LineZ* lineZ;
   6767       lineZ = &sm->linesZ[lz];
   6768       if (lineZ->dict[0] != SVal_INVALID) {
   6769          rcdec_LineZ(lineZ);
   6770          lineZ->dict[0] = SVal_NOACCESS;
   6771          lineZ->dict[1] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID;
   6772       } else {
   6773          clear_LineF_of_Z(lineZ);
   6774       }
   6775       for (UInt i = 0; i < N_LINE_ARANGE/4; i++)
   6776          lineZ->ix2s[i] = 0; /* all refer to dict[0] */
   6777    }
   6778 }
   6779 
   6780 /* Set the given range to SVal_NOACCESS in-place in the secmap.
   6781    a must be cacheline aligned. len must be a multiple of a cacheline
   6782    and must be < N_SECMAP_ARANGE. */
   6783 static void zsm_sset_range_noaccess_in_secmap(Addr a, SizeT len)
   6784 {
   6785    tl_assert (is_valid_scache_tag (a));
   6786    tl_assert (0 == (len & (N_LINE_ARANGE - 1)));
   6787    tl_assert (len < N_SECMAP_ARANGE);
   6788 
   6789    SecMap *sm1 = shmem__find_SecMap (a);
   6790    SecMap *sm2 = shmem__find_SecMap (a + len - 1);
   6791    UWord zix_start = shmem__get_SecMap_offset(a          ) >> N_LINE_BITS;
   6792    UWord zix_end   = shmem__get_SecMap_offset(a + len - 1) >> N_LINE_BITS;
   6793 
   6794    if (sm1) {
   6795       if (CHECK_ZSM) tl_assert(is_sane_SecMap(sm1));
   6796       zsm_secmap_line_range_noaccess (sm1, zix_start,
   6797                                       sm1 == sm2 ? zix_end : N_SECMAP_ZLINES-1);
   6798    }
   6799    if (sm2 && sm1 != sm2) {
   6800       if (CHECK_ZSM) tl_assert(is_sane_SecMap(sm2));
   6801       zsm_secmap_line_range_noaccess (sm2, 0, zix_end);
   6802    }
   6803 }
   6804 
   6805 /* Set the given address range to SVal_NOACCESS.
   6806    The SecMaps fully set to SVal_NOACCESS will be pushed in SecMap_freelist. */
   6807 static void zsm_sset_range_noaccess (Addr addr, SizeT len)
   6808 {
   6809    /*
   6810        BPC = Before, Partial Cacheline, = addr
   6811              (i.e. starting inside a cacheline/inside a SecMap)
   6812        BFC = Before, Full Cacheline(s), but not full SecMap
   6813              (i.e. starting inside a SecMap)
   6814        FSM = Full SecMap(s)
   6815              (i.e. starting a SecMap)
   6816        AFC = After, Full Cacheline(s), but not full SecMap
   6817              (i.e. first address after the full SecMap(s))
   6818        APC = After, Partial Cacheline, i.e. first address after the
   6819              full CacheLines).
   6820        ARE = After Range End = addr+len = first address not part of the range.
   6821 
   6822        If addr     starts a Cacheline, then BPC == BFC.
   6823        If addr     starts a SecMap,    then BPC == BFC == FSM.
   6824        If addr+len starts a SecMap,    then APC == ARE == AFC
   6825        If addr+len starts a Cacheline, then APC == ARE
   6826    */
   6827    Addr ARE = addr + len;
   6828    Addr BPC = addr;
   6829    Addr BFC = ROUNDUP(BPC, N_LINE_ARANGE);
   6830    Addr FSM = ROUNDUP(BPC, N_SECMAP_ARANGE);
   6831    Addr AFC = ROUNDDN(ARE, N_SECMAP_ARANGE);
   6832    Addr APC = ROUNDDN(ARE, N_LINE_ARANGE);
   6833    SizeT Plen = len; // Plen will be split between the following:
   6834    SizeT BPClen;
   6835    SizeT BFClen;
   6836    SizeT FSMlen;
   6837    SizeT AFClen;
   6838    SizeT APClen;
   6839 
   6840    /* Consumes from Plen the nr of bytes between from and to.
   6841       from and to must be aligned on a multiple of round.
   6842       The length consumed will be a multiple of round, with
   6843       a maximum of Plen. */
   6844 #  define PlenCONSUME(from, to, round, consumed) \
   6845    do {                                          \
   6846    if (from < to) {                              \
   6847       if (to - from < Plen)                      \
   6848          consumed = to - from;                   \
   6849       else                                       \
   6850          consumed = ROUNDDN(Plen, round);        \
   6851    } else {                                      \
   6852       consumed = 0;                              \
   6853    }                                             \
   6854    Plen -= consumed; } while (0)
   6855 
   6856    PlenCONSUME(BPC, BFC, 1,               BPClen);
   6857    PlenCONSUME(BFC, FSM, N_LINE_ARANGE,   BFClen);
   6858    PlenCONSUME(FSM, AFC, N_SECMAP_ARANGE, FSMlen);
   6859    PlenCONSUME(AFC, APC, N_LINE_ARANGE,   AFClen);
   6860    PlenCONSUME(APC, ARE, 1,               APClen);
   6861 
   6862    if (0)
   6863       VG_(printf) ("addr %p[%lu] ARE %p"
   6864                    " BPC %p[%lu] BFC %p[%lu] FSM %p[%lu]"
   6865                    " AFC %p[%lu] APC %p[%lu]\n",
   6866                    (void*)addr, len, (void*)ARE,
   6867                    (void*)BPC, BPClen, (void*)BFC, BFClen, (void*)FSM, FSMlen,
   6868                    (void*)AFC, AFClen, (void*)APC, APClen);
   6869 
   6870    tl_assert (Plen == 0);
   6871 
   6872    /* Set to NOACCESS pieces before and after not covered by entire SecMaps. */
   6873 
   6874    /* First we set the partial cachelines. This is done through the cache. */
   6875    if (BPClen > 0)
   6876       zsm_sset_range_SMALL (BPC, BPClen, SVal_NOACCESS);
   6877    if (APClen > 0)
   6878       zsm_sset_range_SMALL (APC, APClen, SVal_NOACCESS);
   6879 
   6880    /* After this, we will not use the cache anymore. We will directly work
   6881       in-place on the z shadow memory in SecMap(s).
   6882       So, we invalidate the cachelines for the whole range we are setting
   6883       to NOACCESS below. */
   6884    shmem__invalidate_scache_range (BFC, APC - BFC);
   6885 
   6886    if (BFClen > 0)
   6887       zsm_sset_range_noaccess_in_secmap (BFC, BFClen);
   6888    if (AFClen > 0)
   6889       zsm_sset_range_noaccess_in_secmap (AFC, AFClen);
   6890 
   6891    if (FSMlen > 0) {
   6892       /* Set to NOACCESS all the SecMaps, pushing the SecMaps to the
   6893          free list. */
   6894       Addr  sm_start = FSM;
   6895       while (sm_start < AFC) {
   6896          SecMap *sm = shmem__find_SecMap (sm_start);
   6897          if (sm) {
   6898             Addr gaKey;
   6899             SecMap *fm_sm;
   6900 
   6901             if (CHECK_ZSM) tl_assert(is_sane_SecMap(sm));
   6902             for (UInt lz = 0; lz < N_SECMAP_ZLINES; lz++) {
   6903                LineZ *lineZ = &sm->linesZ[lz];
   6904                if (LIKELY(lineZ->dict[0] != SVal_INVALID))
   6905                   rcdec_LineZ(lineZ);
   6906                else
   6907                   clear_LineF_of_Z(lineZ);
   6908             }
   6909             if (!VG_(delFromFM)(map_shmem, &gaKey, (UWord*)&fm_sm, sm_start))
   6910                tl_assert (0);
   6911             stats__secmaps_in_map_shmem--;
   6912             tl_assert (gaKey == sm_start);
   6913             tl_assert (sm == fm_sm);
   6914             stats__secmaps_ssetGCed++;
   6915             push_SecMap_on_freelist (sm);
   6916          }
   6917          sm_start += N_SECMAP_ARANGE;
   6918       }
   6919       tl_assert (sm_start == AFC);
   6920 
   6921       /* The above loop might have kept copies of freed SecMap in the smCache.
   6922          => clear them. */
   6923       if (address_in_range(smCache[0].gaKey, FSM, FSMlen)) {
   6924          smCache[0].gaKey = 1;
   6925          smCache[0].sm = NULL;
   6926       }
   6927       if (address_in_range(smCache[1].gaKey, FSM, FSMlen)) {
   6928          smCache[1].gaKey = 1;
   6929          smCache[1].sm = NULL;
   6930       }
   6931       if (address_in_range(smCache[2].gaKey, FSM, FSMlen)) {
   6932          smCache[2].gaKey = 1;
   6933          smCache[2].sm = NULL;
   6934       }
   6935       STATIC_ASSERT (3 == sizeof(smCache)/sizeof(SMCacheEnt));
   6936    }
   6937 }
   6938 
   6939 void libhb_srange_noaccess_AHAE ( Thr* thr, Addr a, SizeT szB )
   6940 {
   6941    /* This really does put the requested range in NoAccess.  It's
   6942       expensive though. */
   6943    SVal sv = SVal_NOACCESS;
   6944    tl_assert(is_sane_SVal_C(sv));
   6945    if (LIKELY(szB < 2 * N_LINE_ARANGE))
   6946       zsm_sset_range_SMALL (a, szB, SVal_NOACCESS);
   6947    else
   6948       zsm_sset_range_noaccess (a, szB);
   6949    Filter__clear_range( thr->filter, a, szB );
   6950 }
   6951 
   6952 /* Works byte at a time. Can be optimised if needed. */
   6953 UWord libhb_srange_get_abits (Addr a, UChar *abits, SizeT len)
   6954 {
   6955    UWord anr = 0; // nr of bytes addressable.
   6956 
   6957    /* Get the accessibility of each byte. Pay attention to not
   6958       create SecMap or LineZ when checking if a byte is addressable.
   6959 
   6960       Note: this is used for client request. Performance deemed not critical.
   6961       So for simplicity, we work byte per byte.
   6962       Performance could be improved  by working with full cachelines
   6963       or with full SecMap, when reaching a cacheline or secmap boundary. */
   6964    for (SizeT i = 0; i < len; i++) {
   6965       SVal       sv = SVal_INVALID;
   6966       Addr       b = a + i;
   6967       Addr       tag = b & ~(N_LINE_ARANGE - 1);
   6968       UWord      wix = (b >> N_LINE_BITS) & (N_WAY_NENT - 1);
   6969       UWord      cloff = get_cacheline_offset(b);
   6970 
   6971       /* Note: we do not use get_cacheline(b) to avoid creating cachelines
   6972          and/or SecMap for non addressable bytes. */
   6973       if (tag == cache_shmem.tags0[wix]) {
   6974          CacheLine copy = cache_shmem.lyns0[wix];
   6975          /* We work on a copy of the cacheline, as we do not want to
   6976             record the client request as a real read.
   6977             The below is somewhat similar to zsm_sapply08__msmcread but
   6978             avoids side effects on the cache. */
   6979          UWord toff = get_tree_offset(b); /* == 0 .. 7 */
   6980          UWord tno  = get_treeno(b);
   6981          UShort descr = copy.descrs[tno];
   6982          if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) {
   6983             SVal* tree = &copy.svals[tno << 3];
   6984             copy.descrs[tno] = pulldown_to_8(tree, toff, descr);
   6985          }
   6986          sv = copy.svals[cloff];
   6987       } else {
   6988          /* Byte not found in the cacheline. Search for a SecMap. */
   6989          SecMap *sm = shmem__find_SecMap(b);
   6990          LineZ *lineZ;
   6991          if (sm == NULL)
   6992             sv = SVal_NOACCESS;
   6993          else {
   6994             UWord zix = shmem__get_SecMap_offset(b) >> N_LINE_BITS;
   6995             lineZ = &sm->linesZ[zix];
   6996             if (lineZ->dict[0] == SVal_INVALID) {
   6997                LineF *lineF = SVal2Ptr(lineZ->dict[1]);
   6998                sv = lineF->w64s[cloff];
   6999             } else {
   7000                UWord ix = read_twobit_array( lineZ->ix2s, cloff );
   7001                sv = lineZ->dict[ix];
   7002             }
   7003          }
   7004       }
   7005 
   7006       tl_assert (sv != SVal_INVALID);
   7007       if (sv == SVal_NOACCESS) {
   7008          if (abits)
   7009             abits[i] = 0x00;
   7010       } else {
   7011          if (abits)
   7012             abits[i] = 0xff;
   7013          anr++;
   7014       }
   7015    }
   7016 
   7017    return anr;
   7018 }
   7019 
   7020 
   7021 void libhb_srange_untrack ( Thr* thr, Addr a, SizeT szB )
   7022 {
   7023    SVal sv = SVal_NOACCESS;
   7024    tl_assert(is_sane_SVal_C(sv));
   7025    if (0 && TRACEME(a,szB)) trace(thr,a,szB,"untrack-before");
   7026    if (LIKELY(szB < 2 * N_LINE_ARANGE))
   7027       zsm_sset_range_SMALL (a, szB, SVal_NOACCESS);
   7028    else
   7029       zsm_sset_range_noaccess (a, szB);
   7030    Filter__clear_range( thr->filter, a, szB );
   7031    if (0 && TRACEME(a,szB)) trace(thr,a,szB,"untrack-after ");
   7032 }
   7033 
   7034 Thread* libhb_get_Thr_hgthread ( Thr* thr ) {
   7035    tl_assert(thr);
   7036    return thr->hgthread;
   7037 }
   7038 
   7039 void libhb_set_Thr_hgthread ( Thr* thr, Thread* hgthread ) {
   7040    tl_assert(thr);
   7041    thr->hgthread = hgthread;
   7042 }
   7043 
   7044 void libhb_copy_shadow_state ( Thr* thr, Addr src, Addr dst, SizeT len )
   7045 {
   7046    zsm_scopy_range(src, dst, len);
   7047    Filter__clear_range( thr->filter, dst, len );
   7048 }
   7049 
   7050 void libhb_maybe_GC ( void )
   7051 {
   7052    /* GC the unreferenced (zero rc) RCECs when
   7053          (1) reaching a significant nr of RCECs (to avoid scanning a contextTab
   7054              with mostly NULL ptr)
   7055      and (2) approaching the max nr of RCEC (as we have in any case
   7056              at least that amount of RCEC in the pool allocator)
   7057              Note: the margin allows to avoid a small but constant increase
   7058              of the max nr of RCEC due to the fact that libhb_maybe_GC is
   7059              not called when the current nr of RCEC exactly reaches the max.
   7060      and (3) the nr of referenced RCECs is less than 75% than total nr RCECs.
   7061      Avoid growing too much the nr of RCEC keeps the memory use low,
   7062      and avoids to have too many elements in the (fixed) contextTab hashtable.
   7063    */
   7064    if (UNLIKELY(stats__ctxt_tab_curr > N_RCEC_TAB/2
   7065                 && stats__ctxt_tab_curr + 1000 >= stats__ctxt_tab_max
   7066                 && (stats__ctxt_tab_curr * 3)/4 > RCEC_referenced))
   7067       do_RCEC_GC();
   7068 
   7069    /* If there are still no entries available (all the table entries are full),
   7070       and we hit the threshold point, then do a GC */
   7071    Bool vts_tab_GC = vts_tab_freelist == VtsID_INVALID
   7072       && VG_(sizeXA)( vts_tab ) >= vts_next_GC_at;
   7073    if (UNLIKELY (vts_tab_GC))
   7074       vts_tab__do_GC( False/*don't show stats*/ );
   7075 
   7076    /* scan GC the SecMaps when
   7077           (1) no SecMap in the freelist
   7078       and (2) the current nr of live secmaps exceeds the threshold. */
   7079    if (UNLIKELY(SecMap_freelist == NULL
   7080                 && stats__secmaps_in_map_shmem >= next_SecMap_GC_at)) {
   7081       // If we did a vts tab GC, then no need to flush the cache again.
   7082       if (!vts_tab_GC)
   7083          zsm_flush_cache();
   7084       shmem__SecMap_do_GC(True);
   7085    }
   7086 
   7087    /* Check the reference counts (expensive) */
   7088    if (CHECK_CEM)
   7089       event_map__check_reference_counts();
   7090 }
   7091 
   7092 
   7093 /////////////////////////////////////////////////////////////////
   7094 /////////////////////////////////////////////////////////////////
   7095 //                                                             //
   7096 // SECTION END main library                                    //
   7097 //                                                             //
   7098 /////////////////////////////////////////////////////////////////
   7099 /////////////////////////////////////////////////////////////////
   7100 
   7101 /*--------------------------------------------------------------------*/
   7102 /*--- end                                             libhb_main.c ---*/
   7103 /*--------------------------------------------------------------------*/
   7104