1 /* 2 * Copyright (c) 2017 Imagination Technologies. 3 * 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer 14 * in the documentation and/or other materials provided with 15 * the distribution. 16 * * Neither the name of Imagination Technologies nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 #include <string.h> 33 34 #define ENABLE_PREFETCH 1 35 #define op_t unsigned long int 36 #define op_size sizeof (op_t) 37 38 #if ENABLE_PREFETCH 39 #define PREFETCH(addr) __builtin_prefetch (addr, 0, 1); 40 #else 41 #define PREFETCH(addr) 42 #endif 43 44 #if __mips64 || __mips_isa_rev >= 2 45 static inline void * __attribute__ ((always_inline)) 46 do_bytes (const op_t* w, op_t inval) 47 { 48 const unsigned char *p = (const unsigned char *) w; 49 op_t outval = 0; 50 #if __mips64 51 __asm__ volatile ( 52 "dsbh %1, %0 \n\t" 53 "dshd %0, %1 \n\t" 54 "dclz %1, %0 \n\t" 55 : "+r" (inval), "+r" (outval) 56 ); 57 #else 58 __asm__ volatile ( 59 "wsbh %1, %0 \n\t" 60 "rotr %0, %1, 16 \n\t" 61 "clz %1, %0 \n\t" 62 : "+r" (inval), "+r" (outval) 63 ); 64 #endif 65 p += (outval >> 3); 66 return (void *) p; 67 } 68 69 #define DO_WORD(in, val) { \ 70 op_t tmp = ((val - mask_1) & ~val) & mask_128; \ 71 if (tmp != 0) \ 72 return do_bytes(in, tmp); \ 73 } 74 #else 75 static inline void * __attribute__ ((always_inline)) 76 do_bytes (const op_t* w, unsigned char ch) 77 { 78 const unsigned char *p = (const unsigned char *) w; 79 for (; *p != ch; ++p); 80 return (void *) p; 81 } 82 83 #define DO_WORD(in, val) { \ 84 op_t tmp = ((val - mask_1) & ~val) & mask_128; \ 85 if (tmp != 0) \ 86 return do_bytes(in, ch); \ 87 } 88 #endif 89 90 #define DO_WORDS(w) { \ 91 op_t* w1 = (op_t*) w; \ 92 op_t val0 = w1[0] ^ mask_c; \ 93 op_t val1 = w1[1] ^ mask_c; \ 94 op_t val2 = w1[2] ^ mask_c; \ 95 op_t val3 = w1[3] ^ mask_c; \ 96 DO_WORD(w1, val0) \ 97 DO_WORD(w1 + 1, val1) \ 98 DO_WORD(w1 + 2, val2) \ 99 DO_WORD(w1 + 3, val3) \ 100 } 101 102 void * 103 memchr (void const *s, int c_in, size_t n) 104 { 105 if (n != 0) { 106 const unsigned char *p = (const unsigned char *) s; 107 const op_t *w; 108 op_t mask_1, mask_128, mask_c; 109 unsigned char ch = (unsigned char) c_in; 110 111 /* 112 * Check bytewize till initial alignment 113 */ 114 for (; n > 0 && ((size_t) p % op_size) != 0; --n, ++p) { 115 if (*p == ch) 116 return (void *) p; 117 } 118 119 w = (const op_t *) p; 120 121 mask_c = ch | (ch << 8); 122 mask_c |= mask_c << 16; 123 __asm__ volatile ( 124 "li %0, 0x01010101 \n\t" 125 : "=r" (mask_1) 126 ); 127 #if __mips64 128 mask_1 |= mask_1 << 32; 129 mask_c |= mask_c << 32; 130 #endif 131 mask_128 = mask_1 << 7; 132 133 /* 134 * Check op_size byteswize after initial alignment 135 */ 136 #if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400) 137 PREFETCH (w); 138 PREFETCH (w + 8); 139 while (n >= 24 * op_size) { 140 PREFETCH(w + 16); 141 DO_WORDS(w); 142 DO_WORDS(w + 4); 143 w += 8; 144 n -= 8 * op_size; 145 } 146 while (n >= 8 * op_size) { 147 DO_WORDS(w); 148 DO_WORDS(w + 4); 149 w += 8; 150 n -= 8 * op_size; 151 } 152 #else 153 PREFETCH (w); 154 PREFETCH (w + 4); 155 while (n >= 12 * op_size) { 156 PREFETCH(w + 8); 157 DO_WORDS(w); 158 w += 4; 159 n -= 4 * op_size; 160 } 161 while (n >= 4 * op_size) { 162 DO_WORDS(w); 163 w += 4; 164 n -= 4 * op_size; 165 } 166 #endif 167 168 while (n >= op_size) { 169 op_t val = *w ^ mask_c; 170 DO_WORD(w, val); 171 w++; 172 n -= op_size; 173 } 174 175 /* 176 * Check bytewize for remaining bytes 177 */ 178 p = (const unsigned char *) w; 179 for (; n > 0; --n, ++p) { 180 if (*p == ch) 181 return (void *) p; 182 } 183 } 184 return NULL; 185 } 186
