Home | History | Annotate | Download | only in text 
      1 /*
      2  * Copyright (C) 2011, 2012 Apple Inc. All rights reserved.
      3  * Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
      4  *
      5  * This library is free software; you can redistribute it and/or
      6  * modify it under the terms of the GNU Library General Public
      7  * License as published by the Free Software Foundation; either
      8  * version 2 of the License, or (at your option) any later version.
      9  *
     10  * This library is distributed in the hope that it will be useful,
     11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
     12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     13  * Library General Public License for more details.
     14  *
     15  * You should have received a copy of the GNU Library General Public License
     16  * along with this library; see the file COPYING.LIB.  If not, write to
     17  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     18  * Boston, MA 02110-1301, USA.
     19  *
     20  */
     21 
     22 #ifndef ASCIIFastPath_h
     23 #define ASCIIFastPath_h
     24 
     25 #include "wtf/Alignment.h"
     26 #include "wtf/CPU.h"
     27 #include "wtf/StdLibExtras.h"
     28 #include "wtf/unicode/Unicode.h"
     29 #include <stdint.h>
     30 
     31 #if OS(MACOSX) && (CPU(X86) || CPU(X86_64))
     32 #include <emmintrin.h>
     33 #endif
     34 
     35 namespace WTF {
     36 
     37 // Assuming that a pointer is the size of a "machine word", then
     38 // uintptr_t is an integer type that is also a machine word.
     39 typedef uintptr_t MachineWord;
     40 const uintptr_t machineWordAlignmentMask = sizeof(MachineWord) - 1;
     41 
     42 inline bool isAlignedToMachineWord(const void* pointer)
     43 {
     44     return !(reinterpret_cast<uintptr_t>(pointer) & machineWordAlignmentMask);
     45 }
     46 
     47 template<typename T> inline T* alignToMachineWord(T* pointer)
     48 {
     49     return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(pointer) & ~machineWordAlignmentMask);
     50 }
     51 
     52 template<size_t size, typename CharacterType> struct NonASCIIMask;
     53 template<> struct NonASCIIMask<4, UChar> {
     54     static inline uint32_t value() { return 0xFF80FF80U; }
     55 };
     56 template<> struct NonASCIIMask<4, LChar> {
     57     static inline uint32_t value() { return 0x80808080U; }
     58 };
     59 template<> struct NonASCIIMask<8, UChar> {
     60     static inline uint64_t value() { return 0xFF80FF80FF80FF80ULL; }
     61 };
     62 template<> struct NonASCIIMask<8, LChar> {
     63     static inline uint64_t value() { return 0x8080808080808080ULL; }
     64 };
     65 
     66 
     67 template<typename CharacterType>
     68 inline bool isAllASCII(MachineWord word)
     69 {
     70     return !(word & NonASCIIMask<sizeof(MachineWord), CharacterType>::value());
     71 }
     72 
     73 // Note: This function assume the input is likely all ASCII, and
     74 // does not leave early if it is not the case.
     75 template<typename CharacterType>
     76 inline bool charactersAreAllASCII(const CharacterType* characters, size_t length)
     77 {
     78     MachineWord allCharBits = 0;
     79     const CharacterType* end = characters + length;
     80 
     81     // Prologue: align the input.
     82     while (!isAlignedToMachineWord(characters) && characters != end) {
     83         allCharBits |= *characters;
     84         ++characters;
     85     }
     86 
     87     // Compare the values of CPU word size.
     88     const CharacterType* wordEnd = alignToMachineWord(end);
     89     const size_t loopIncrement = sizeof(MachineWord) / sizeof(CharacterType);
     90     while (characters < wordEnd) {
     91         allCharBits |= *(reinterpret_cast_ptr<const MachineWord*>(characters));
     92         characters += loopIncrement;
     93     }
     94 
     95     // Process the remaining bytes.
     96     while (characters != end) {
     97         allCharBits |= *characters;
     98         ++characters;
     99     }
    100 
    101     MachineWord nonASCIIBitMask = NonASCIIMask<sizeof(MachineWord), CharacterType>::value();
    102     return !(allCharBits & nonASCIIBitMask);
    103 }
    104 
    105 inline void copyLCharsFromUCharSource(LChar* destination, const UChar* source, size_t length)
    106 {
    107 #if OS(MACOSX) && (CPU(X86) || CPU(X86_64))
    108     const uintptr_t memoryAccessSize = 16; // Memory accesses on 16 byte (128 bit) alignment
    109     const uintptr_t memoryAccessMask = memoryAccessSize - 1;
    110 
    111     size_t i = 0;
    112     for (;i < length && !isAlignedTo<memoryAccessMask>(&source[i]); ++i) {
    113         ASSERT(!(source[i] & 0xff00));
    114         destination[i] = static_cast<LChar>(source[i]);
    115     }
    116 
    117     const uintptr_t sourceLoadSize = 32; // Process 32 bytes (16 UChars) each iteration
    118     const size_t ucharsPerLoop = sourceLoadSize / sizeof(UChar);
    119     if (length > ucharsPerLoop) {
    120         const size_t endLength = length - ucharsPerLoop + 1;
    121         for (; i < endLength; i += ucharsPerLoop) {
    122 #ifndef NDEBUG
    123             for (unsigned checkIndex = 0; checkIndex < ucharsPerLoop; ++checkIndex)
    124                 ASSERT(!(source[i+checkIndex] & 0xff00));
    125 #endif
    126             __m128i first8UChars = _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i]));
    127             __m128i second8UChars = _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i+8]));
    128             __m128i packedChars = _mm_packus_epi16(first8UChars, second8UChars);
    129             _mm_storeu_si128(reinterpret_cast<__m128i*>(&destination[i]), packedChars);
    130         }
    131     }
    132 
    133     for (; i < length; ++i) {
    134         ASSERT(!(source[i] & 0xff00));
    135         destination[i] = static_cast<LChar>(source[i]);
    136     }
    137 #elif COMPILER(GCC) && CPU(ARM_NEON) && !(CPU(BIG_ENDIAN) || CPU(MIDDLE_ENDIAN)) && defined(NDEBUG)
    138     const LChar* const end = destination + length;
    139     const uintptr_t memoryAccessSize = 8;
    140 
    141     if (length >= (2 * memoryAccessSize) - 1) {
    142         // Prefix: align dst on 64 bits.
    143         const uintptr_t memoryAccessMask = memoryAccessSize - 1;
    144         while (!isAlignedTo<memoryAccessMask>(destination))
    145             *destination++ = static_cast<LChar>(*source++);
    146 
    147         // Vector interleaved unpack, we only store the lower 8 bits.
    148         const uintptr_t lengthLeft = end - destination;
    149         const LChar* const simdEnd = end - (lengthLeft % memoryAccessSize);
    150         do {
    151             asm("vld2.8   { d0-d1 }, [%[SOURCE]] !\n\t"
    152                 "vst1.8   { d0 }, [%[DESTINATION],:64] !\n\t"
    153                 : [SOURCE]"+r" (source), [DESTINATION]"+r" (destination)
    154                 :
    155                 : "memory", "d0", "d1");
    156         } while (destination != simdEnd);
    157     }
    158 
    159     while (destination != end)
    160         *destination++ = static_cast<LChar>(*source++);
    161 #else
    162     for (size_t i = 0; i < length; ++i) {
    163         ASSERT(!(source[i] & 0xff00));
    164         destination[i] = static_cast<LChar>(source[i]);
    165     }
    166 #endif
    167 }
    168 
    169 } // namespace WTF
    170 
    171 #endif // ASCIIFastPath_h
    172