1 /* 2 * Copyright (c) 2017 Imagination Technologies. 3 * 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer 14 * in the documentation and/or other materials provided with 15 * the distribution. 16 * * Neither the name of Imagination Technologies nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <string.h> 34 35 #if !defined(UNALIGNED_INSTR_SUPPORT) 36 /* does target have unaligned lw/ld/ualw/uald instructions? */ 37 #define UNALIGNED_INSTR_SUPPORT 0 38 #if __mips_isa_rev < 6 && !__mips1 39 #undef UNALIGNED_INSTR_SUPPORT 40 #define UNALIGNED_INSTR_SUPPORT 1 41 #endif 42 #endif 43 44 #if !defined(HW_UNALIGNED_SUPPORT) 45 /* Does target have hardware support for unaligned accesses? */ 46 #define HW_UNALIGNED_SUPPORT 0 47 #if __mips_isa_rev >= 6 48 #undef HW_UNALIGNED_SUPPORT 49 #define HW_UNALIGNED_SUPPORT 1 50 #endif 51 #endif 52 53 #define ENABLE_PREFETCH 1 54 55 #if ENABLE_PREFETCH 56 #define PREFETCH(addr) __builtin_prefetch (addr, 0, 1); 57 #else 58 #define PREFETCH(addr) 59 #endif 60 61 #if _MIPS_SIM == _ABIO32 62 typedef unsigned long reg_t; 63 typedef struct 64 { 65 reg_t B0:8, B1:8, B2:8, B3:8; 66 } bits_t; 67 #else 68 typedef unsigned long long reg_t; 69 typedef struct 70 { 71 reg_t B0:8, B1:8, B2:8, B3:8, B4:8, B5:8, B6:8, B7:8; 72 } bits_t; 73 #endif 74 75 typedef union 76 { 77 reg_t v; 78 bits_t b; 79 } bitfields_t; 80 81 #define DO_BYTE(a, i) \ 82 a[i] = bw.b.B##i; \ 83 len--; \ 84 if(!len) return ret; \ 85 86 /* This code is called when aligning a pointer, there are remaining bytes 87 after doing word compares, or architecture does not have some form 88 of unaligned support. */ 89 static inline void * __attribute__ ((always_inline)) 90 do_bytes (void *a, const void *b, unsigned long len, void *ret) 91 { 92 unsigned char *x = (unsigned char *) a; 93 unsigned char *y = (unsigned char *) b; 94 unsigned long i; 95 96 /* 'len' might be zero here, so preloading the first two values 97 before the loop may access unallocated memory. */ 98 for (i = 0; i < len; i++) 99 { 100 *x = *y; 101 x++; 102 y++; 103 } 104 return ret; 105 } 106 107 static inline void * __attribute__ ((always_inline)) 108 do_bytes_backward (void *a, const void *b, unsigned long len, void *ret) 109 { 110 unsigned char *x = (unsigned char *) a; 111 unsigned char *y = (unsigned char *) b; 112 unsigned long i; 113 114 /* 'len' might be zero here, so preloading the first two values 115 before the loop may access unallocated memory. */ 116 for (i = 0; i < len; i++) { 117 *--x = *--y; 118 } 119 return ret; 120 } 121 122 static inline void * __attribute__ ((always_inline)) 123 do_bytes_aligned (void *a, const void *b, unsigned long len, void *ret) 124 { 125 unsigned char *x = (unsigned char *) a; 126 127 if(len > 0) { 128 bitfields_t bw; 129 bw.v = *((reg_t*) b); 130 131 #if __mips64 132 DO_BYTE(x, 0); 133 DO_BYTE(x, 1); 134 DO_BYTE(x, 2); 135 DO_BYTE(x, 3); 136 DO_BYTE(x, 4); 137 DO_BYTE(x, 5); 138 DO_BYTE(x, 6); 139 DO_BYTE(x, 7); 140 #else 141 DO_BYTE(x, 0); 142 DO_BYTE(x, 1); 143 DO_BYTE(x, 2); 144 DO_BYTE(x, 3); 145 #endif 146 } 147 148 return ret; 149 } 150 151 #if !HW_UNALIGNED_SUPPORT 152 #if UNALIGNED_INSTR_SUPPORT 153 /* for MIPS GCC, there are no unaligned builtins - so this struct forces 154 the compiler to treat the pointer access as unaligned. */ 155 struct ulw 156 { 157 reg_t uli; 158 } __attribute__ ((packed)); 159 160 #define STORE_UNALIGNED_8(a, b) \ 161 { \ 162 reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3]; \ 163 reg_t y4 = b[4], y5 = b[5], y6 = b[6], y7 = b[7]; \ 164 a[0].uli = y0; \ 165 a[1].uli = y1; \ 166 a[2].uli = y2; \ 167 a[3].uli = y3; \ 168 a[4].uli = y4; \ 169 a[5].uli = y5; \ 170 a[6].uli = y6; \ 171 a[7].uli = y7; \ 172 } 173 174 #define STORE_UNALIGNED_4(a, b) \ 175 { \ 176 reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3]; \ 177 a[0].uli = y0; \ 178 a[1].uli = y1; \ 179 a[2].uli = y2; \ 180 a[3].uli = y3; \ 181 } 182 183 /* first pointer is not aligned while second pointer is. */ 184 static void * 185 unaligned_words_forward (struct ulw *a, const reg_t * b, 186 unsigned long words, unsigned long bytes, void *ret) 187 { 188 #if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400) 189 unsigned long i, words_by_8, words_by_1; 190 words_by_1 = words % 8; 191 words_by_8 = words >> 3; 192 for (; words_by_8 > 0; words_by_8--) { 193 if(words_by_8 != 1) 194 PREFETCH (b + 8); 195 STORE_UNALIGNED_8(a, b); 196 a += 8; 197 b += 8; 198 } 199 #else 200 unsigned long i, words_by_4, words_by_1; 201 words_by_1 = words % 4; 202 words_by_4 = words >> 2; 203 for (; words_by_4 > 0; words_by_4--) { 204 if(words_by_4 != 1) 205 PREFETCH (b + 4); 206 STORE_UNALIGNED_4(a, b); 207 a += 4; 208 b += 4; 209 } 210 #endif 211 212 /* do remaining words. */ 213 for (i = 0; i < words_by_1; i++) { 214 a->uli = *b; 215 a += 1; 216 b += 1; 217 } 218 219 /* mop up any remaining bytes. */ 220 return do_bytes_aligned (a, b, bytes, ret); 221 } 222 223 static void * 224 unaligned_words_backward (struct ulw *a, const reg_t * b, 225 unsigned long words, unsigned long bytes, void *ret) 226 { 227 #if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400) 228 unsigned long i, words_by_8, words_by_1; 229 words_by_1 = words % 8; 230 words_by_8 = words >> 3; 231 for (; words_by_8 > 0; words_by_8--) { 232 if(words_by_8 != 1) 233 PREFETCH (b - 16); 234 a -= 8; 235 b -= 8; 236 STORE_UNALIGNED_8(a, b); 237 } 238 #else 239 unsigned long i, words_by_4, words_by_1; 240 words_by_1 = words % 4; 241 words_by_4 = words >> 2; 242 for (; words_by_4 > 0; words_by_4--) { 243 if(words_by_4 != 1) 244 PREFETCH (b - 8); 245 a -= 4; 246 b -= 4; 247 STORE_UNALIGNED_4(a, b); 248 } 249 #endif 250 251 /* do remaining words. */ 252 for (i = 0; i < words_by_1; i++) { 253 a -= 1; 254 b -= 1; 255 a->uli = *b; 256 } 257 258 /* mop up any remaining bytes. */ 259 return do_bytes_backward (a, b, bytes, ret); 260 } 261 262 #else 263 /* no HW support or unaligned lw/ld/ualw/uald instructions. */ 264 static void * 265 unaligned_words_forward (reg_t * a, const reg_t * b, 266 unsigned long words, unsigned long bytes, void *ret) 267 { 268 return do_bytes_aligned (a, b, (sizeof (reg_t) * words) + bytes, ret); 269 } 270 271 static void * 272 unaligned_words_backward (reg_t * a, const reg_t * b, 273 unsigned long words, unsigned long bytes, void *ret) 274 { 275 return do_bytes_backward (a, b, (sizeof (reg_t) * words) + bytes, ret); 276 } 277 278 #endif /* UNALIGNED_INSTR_SUPPORT */ 279 #endif /* HW_UNALIGNED_SUPPORT */ 280 281 /* both pointers are aligned, or first isn't and HW support for unaligned. */ 282 283 #define STORE_ALIGNED_8(a, b) \ 284 { \ 285 reg_t x0 = b[0], x1 = b[1], x2 = b[2], x3 = b[3]; \ 286 reg_t x4 = b[4], x5 = b[5], x6 = b[6], x7 = b[7]; \ 287 a[0] = x0; \ 288 a[1] = x1; \ 289 a[2] = x2; \ 290 a[3] = x3; \ 291 a[4] = x4; \ 292 a[5] = x5; \ 293 a[6] = x6; \ 294 a[7] = x7; \ 295 } 296 297 #define STORE_ALIGNED_4(a, b) \ 298 { \ 299 reg_t x0 = b[0], x1 = b[1], x2 = b[2], x3 = b[3]; \ 300 a[0] = x0; \ 301 a[1] = x1; \ 302 a[2] = x2; \ 303 a[3] = x3; \ 304 } 305 306 static void * 307 aligned_words_forward (reg_t * a, const reg_t * b, 308 unsigned long words, unsigned long bytes, void *ret) 309 { 310 #if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400) 311 unsigned long i, words_by_8, words_by_1; 312 words_by_1 = words % 8; 313 words_by_8 = words >> 3; 314 for (; words_by_8 > 0; words_by_8--) { 315 if(words_by_8 != 1) 316 PREFETCH (b + 8); 317 STORE_ALIGNED_8(a, b); 318 a += 8; 319 b += 8; 320 } 321 #else 322 unsigned long i, words_by_4, words_by_1; 323 words_by_1 = words % 4; 324 words_by_4 = words >> 2; 325 for (; words_by_4 > 0; words_by_4--) { 326 if(words_by_4 != 1) 327 PREFETCH (b + 4); 328 STORE_ALIGNED_4(a, b); 329 a += 4; 330 b += 4; 331 } 332 #endif 333 334 /* do remaining words. */ 335 for (i = 0; i < words_by_1; i++) { 336 *a = *b; 337 a += 1; 338 b += 1; 339 } 340 341 /* mop up any remaining bytes. */ 342 return do_bytes_aligned (a, b, bytes, ret); 343 } 344 345 346 static void * 347 aligned_words_backward (reg_t * a, const reg_t * b, 348 unsigned long words, unsigned long bytes, void *ret) 349 { 350 #if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400) 351 unsigned long i, words_by_8, words_by_1; 352 words_by_1 = words % 8; 353 words_by_8 = words >> 3; 354 for (; words_by_8 > 0; words_by_8--) { 355 if(words_by_8 != 1) 356 PREFETCH (b - 16); 357 a -= 8; 358 b -= 8; 359 STORE_ALIGNED_8(a, b); 360 } 361 #else 362 unsigned long i, words_by_4, words_by_1; 363 words_by_1 = words % 4; 364 words_by_4 = words >> 2; 365 for (; words_by_4 > 0; words_by_4--) { 366 if(words_by_4 != 1) 367 PREFETCH (b - 8); 368 a -= 4; 369 b -= 4; 370 STORE_ALIGNED_4(a, b); 371 } 372 #endif 373 374 /* do remaining words. */ 375 for (i = 0; i < words_by_1; i++) { 376 a -= 1; 377 b -= 1; 378 *a = *b; 379 } 380 381 /* mop up any remaining bytes. */ 382 return do_bytes_backward (a, b, bytes, ret); 383 } 384 385 void * 386 memmove (void *dst0, const void *src0, size_t length) 387 { 388 unsigned long bytes, words; 389 void *ret = dst0; 390 391 if (length == 0 || dst0 == src0) /* nothing to do */ 392 return dst0; 393 394 if ((unsigned long)dst0 < (unsigned long)src0) { 395 /* Copy forwards. */ 396 /* This shouldn't hit that often. */ 397 if (length < sizeof (reg_t) * 4) { 398 return do_bytes (dst0, src0, length, ret); 399 } 400 401 /* Align the second pointer to word/dword alignment. 402 Note that the pointer is only 32-bits for o32/n32 ABIs. For 403 n32, loads are done as 64-bit while address remains 32-bit. */ 404 bytes = ((unsigned long) src0) % sizeof (reg_t); 405 if (bytes) { 406 bytes = sizeof (reg_t) - bytes; 407 if (bytes > length) 408 bytes = length; 409 do_bytes (dst0, src0, bytes, ret); 410 if (length == bytes) 411 return ret; 412 length -= bytes; 413 dst0 = (void *) (((unsigned char *) dst0) + bytes); 414 src0 = (const void *) (((unsigned char *) src0) + bytes); 415 } 416 417 /* Second pointer now aligned. */ 418 words = length / sizeof (reg_t); 419 bytes = length % sizeof (reg_t); 420 #if HW_UNALIGNED_SUPPORT 421 /* treat possible unaligned first pointer as aligned. */ 422 return aligned_words_forward (dst0, src0, words, bytes, ret); 423 #else 424 if (((unsigned long) dst0) % sizeof (reg_t) == 0) { 425 return aligned_words_forward (dst0, src0, words, bytes, ret); 426 } 427 /* need to use unaligned instructions on first pointer. */ 428 return unaligned_words_forward (dst0, src0, words, bytes, ret); 429 #endif 430 } else { 431 /* Copy backwards. */ 432 dst0 = (void *) (((unsigned char *) dst0) + length); 433 src0 = (const void *) (((unsigned char *) src0) + length); 434 435 /* This shouldn't hit that often. */ 436 if (length < sizeof (reg_t) * 4) { 437 return do_bytes_backward (dst0, src0, length, ret); 438 } 439 440 /* Align the second pointer to word/dword alignment. 441 Note that the pointer is only 32-bits for o32/n32 ABIs. For 442 n32, loads are done as 64-bit while address remains 32-bit. */ 443 bytes = ((unsigned long) src0) % sizeof (reg_t); 444 if (bytes) { 445 if (bytes > length) 446 bytes = length; 447 do_bytes_backward (dst0, src0, bytes, ret); 448 if (length == bytes) 449 return ret; 450 length -= bytes; 451 dst0 = (void *) (((unsigned char *) dst0) - bytes); 452 src0 = (const void *) (((unsigned char *) src0) - bytes); 453 } 454 455 words = length / sizeof (reg_t); 456 bytes = length % sizeof (reg_t); 457 #if HW_UNALIGNED_SUPPORT 458 /* treat possible unaligned first pointer as aligned. */ 459 return aligned_words_backward ((void *)dst0, (void *)src0, words, bytes, ret); 460 #else 461 if (((unsigned long) dst0) % sizeof (reg_t) == 0) { 462 return aligned_words_backward (dst0, src0, words, bytes, ret); 463 } 464 /* need to use unaligned instructions on first pointer. */ 465 return unaligned_words_backward (dst0, src0, words, bytes, ret); 466 #endif 467 } 468 } 469
