1 // Copyright 2012 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/arm/codegen-arm.h" 6 7 #if V8_TARGET_ARCH_ARM 8 9 #include <memory> 10 11 #include "src/arm/simulator-arm.h" 12 #include "src/codegen.h" 13 #include "src/macro-assembler.h" 14 15 namespace v8 { 16 namespace internal { 17 18 19 #define __ masm. 20 21 #if defined(V8_HOST_ARCH_ARM) 22 MemCopyUint8Function CreateMemCopyUint8Function(Isolate* isolate, 23 MemCopyUint8Function stub) { 24 #if defined(USE_SIMULATOR) 25 return stub; 26 #else 27 size_t actual_size; 28 byte* buffer = 29 static_cast<byte*>(base::OS::Allocate(1 * KB, &actual_size, true)); 30 if (buffer == nullptr) return stub; 31 32 MacroAssembler masm(isolate, buffer, static_cast<int>(actual_size), 33 CodeObjectRequired::kNo); 34 35 Register dest = r0; 36 Register src = r1; 37 Register chars = r2; 38 Register temp1 = r3; 39 Label less_4; 40 41 if (CpuFeatures::IsSupported(NEON)) { 42 CpuFeatureScope scope(&masm, NEON); 43 Label loop, less_256, less_128, less_64, less_32, _16_or_less, _8_or_less; 44 Label size_less_than_8; 45 __ pld(MemOperand(src, 0)); 46 47 __ cmp(chars, Operand(8)); 48 __ b(lt, &size_less_than_8); 49 __ cmp(chars, Operand(32)); 50 __ b(lt, &less_32); 51 if (CpuFeatures::dcache_line_size() == 32) { 52 __ pld(MemOperand(src, 32)); 53 } 54 __ cmp(chars, Operand(64)); 55 __ b(lt, &less_64); 56 __ pld(MemOperand(src, 64)); 57 if (CpuFeatures::dcache_line_size() == 32) { 58 __ pld(MemOperand(src, 96)); 59 } 60 __ cmp(chars, Operand(128)); 61 __ b(lt, &less_128); 62 __ pld(MemOperand(src, 128)); 63 if (CpuFeatures::dcache_line_size() == 32) { 64 __ pld(MemOperand(src, 160)); 65 } 66 __ pld(MemOperand(src, 192)); 67 if (CpuFeatures::dcache_line_size() == 32) { 68 __ pld(MemOperand(src, 224)); 69 } 70 __ cmp(chars, Operand(256)); 71 __ b(lt, &less_256); 72 __ sub(chars, chars, Operand(256)); 73 74 __ bind(&loop); 75 __ pld(MemOperand(src, 256)); 76 __ vld1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(src, PostIndex)); 77 if (CpuFeatures::dcache_line_size() == 32) { 78 __ pld(MemOperand(src, 256)); 79 } 80 __ vld1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(src, PostIndex)); 81 __ sub(chars, chars, Operand(64), SetCC); 82 __ vst1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(dest, PostIndex)); 83 __ vst1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(dest, PostIndex)); 84 __ b(ge, &loop); 85 __ add(chars, chars, Operand(256)); 86 87 __ bind(&less_256); 88 __ vld1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(src, PostIndex)); 89 __ vld1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(src, PostIndex)); 90 __ sub(chars, chars, Operand(128)); 91 __ vst1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(dest, PostIndex)); 92 __ vst1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(dest, PostIndex)); 93 __ vld1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(src, PostIndex)); 94 __ vld1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(src, PostIndex)); 95 __ vst1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(dest, PostIndex)); 96 __ vst1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(dest, PostIndex)); 97 __ cmp(chars, Operand(64)); 98 __ b(lt, &less_64); 99 100 __ bind(&less_128); 101 __ vld1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(src, PostIndex)); 102 __ vld1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(src, PostIndex)); 103 __ sub(chars, chars, Operand(64)); 104 __ vst1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(dest, PostIndex)); 105 __ vst1(Neon8, NeonListOperand(d4, 4), NeonMemOperand(dest, PostIndex)); 106 107 __ bind(&less_64); 108 __ cmp(chars, Operand(32)); 109 __ b(lt, &less_32); 110 __ vld1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(src, PostIndex)); 111 __ vst1(Neon8, NeonListOperand(d0, 4), NeonMemOperand(dest, PostIndex)); 112 __ sub(chars, chars, Operand(32)); 113 114 __ bind(&less_32); 115 __ cmp(chars, Operand(16)); 116 __ b(le, &_16_or_less); 117 __ vld1(Neon8, NeonListOperand(d0, 2), NeonMemOperand(src, PostIndex)); 118 __ vst1(Neon8, NeonListOperand(d0, 2), NeonMemOperand(dest, PostIndex)); 119 __ sub(chars, chars, Operand(16)); 120 121 __ bind(&_16_or_less); 122 __ cmp(chars, Operand(8)); 123 __ b(le, &_8_or_less); 124 __ vld1(Neon8, NeonListOperand(d0), NeonMemOperand(src, PostIndex)); 125 __ vst1(Neon8, NeonListOperand(d0), NeonMemOperand(dest, PostIndex)); 126 __ sub(chars, chars, Operand(8)); 127 128 // Do a last copy which may overlap with the previous copy (up to 8 bytes). 129 __ bind(&_8_or_less); 130 __ rsb(chars, chars, Operand(8)); 131 __ sub(src, src, Operand(chars)); 132 __ sub(dest, dest, Operand(chars)); 133 __ vld1(Neon8, NeonListOperand(d0), NeonMemOperand(src)); 134 __ vst1(Neon8, NeonListOperand(d0), NeonMemOperand(dest)); 135 136 __ Ret(); 137 138 __ bind(&size_less_than_8); 139 140 __ bic(temp1, chars, Operand(0x3), SetCC); 141 __ b(&less_4, eq); 142 __ ldr(temp1, MemOperand(src, 4, PostIndex)); 143 __ str(temp1, MemOperand(dest, 4, PostIndex)); 144 } else { 145 Register temp2 = ip; 146 Label loop; 147 148 __ bic(temp2, chars, Operand(0x3), SetCC); 149 __ b(&less_4, eq); 150 __ add(temp2, dest, temp2); 151 152 __ bind(&loop); 153 __ ldr(temp1, MemOperand(src, 4, PostIndex)); 154 __ str(temp1, MemOperand(dest, 4, PostIndex)); 155 __ cmp(dest, temp2); 156 __ b(&loop, ne); 157 } 158 159 __ bind(&less_4); 160 __ mov(chars, Operand(chars, LSL, 31), SetCC); 161 // bit0 => Z (ne), bit1 => C (cs) 162 __ ldrh(temp1, MemOperand(src, 2, PostIndex), cs); 163 __ strh(temp1, MemOperand(dest, 2, PostIndex), cs); 164 __ ldrb(temp1, MemOperand(src), ne); 165 __ strb(temp1, MemOperand(dest), ne); 166 __ Ret(); 167 168 CodeDesc desc; 169 masm.GetCode(&desc); 170 DCHECK(!RelocInfo::RequiresRelocation(desc)); 171 172 Assembler::FlushICache(isolate, buffer, actual_size); 173 base::OS::ProtectCode(buffer, actual_size); 174 return FUNCTION_CAST<MemCopyUint8Function>(buffer); 175 #endif 176 } 177 178 179 // Convert 8 to 16. The number of character to copy must be at least 8. 180 MemCopyUint16Uint8Function CreateMemCopyUint16Uint8Function( 181 Isolate* isolate, MemCopyUint16Uint8Function stub) { 182 #if defined(USE_SIMULATOR) 183 return stub; 184 #else 185 size_t actual_size; 186 byte* buffer = 187 static_cast<byte*>(base::OS::Allocate(1 * KB, &actual_size, true)); 188 if (buffer == nullptr) return stub; 189 190 MacroAssembler masm(isolate, buffer, static_cast<int>(actual_size), 191 CodeObjectRequired::kNo); 192 193 Register dest = r0; 194 Register src = r1; 195 Register chars = r2; 196 if (CpuFeatures::IsSupported(NEON)) { 197 CpuFeatureScope scope(&masm, NEON); 198 Register temp = r3; 199 Label loop; 200 201 __ bic(temp, chars, Operand(0x7)); 202 __ sub(chars, chars, Operand(temp)); 203 __ add(temp, dest, Operand(temp, LSL, 1)); 204 205 __ bind(&loop); 206 __ vld1(Neon8, NeonListOperand(d0), NeonMemOperand(src, PostIndex)); 207 __ vmovl(NeonU8, q0, d0); 208 __ vst1(Neon16, NeonListOperand(d0, 2), NeonMemOperand(dest, PostIndex)); 209 __ cmp(dest, temp); 210 __ b(&loop, ne); 211 212 // Do a last copy which will overlap with the previous copy (1 to 8 bytes). 213 __ rsb(chars, chars, Operand(8)); 214 __ sub(src, src, Operand(chars)); 215 __ sub(dest, dest, Operand(chars, LSL, 1)); 216 __ vld1(Neon8, NeonListOperand(d0), NeonMemOperand(src)); 217 __ vmovl(NeonU8, q0, d0); 218 __ vst1(Neon16, NeonListOperand(d0, 2), NeonMemOperand(dest)); 219 __ Ret(); 220 } else { 221 Register temp1 = r3; 222 Register temp2 = ip; 223 Register temp3 = lr; 224 Register temp4 = r4; 225 Label loop; 226 Label not_two; 227 228 __ Push(lr, r4); 229 __ bic(temp2, chars, Operand(0x3)); 230 __ add(temp2, dest, Operand(temp2, LSL, 1)); 231 232 __ bind(&loop); 233 __ ldr(temp1, MemOperand(src, 4, PostIndex)); 234 __ uxtb16(temp3, temp1); 235 __ uxtb16(temp4, temp1, 8); 236 __ pkhbt(temp1, temp3, Operand(temp4, LSL, 16)); 237 __ str(temp1, MemOperand(dest)); 238 __ pkhtb(temp1, temp4, Operand(temp3, ASR, 16)); 239 __ str(temp1, MemOperand(dest, 4)); 240 __ add(dest, dest, Operand(8)); 241 __ cmp(dest, temp2); 242 __ b(&loop, ne); 243 244 __ mov(chars, Operand(chars, LSL, 31), SetCC); // bit0 => ne, bit1 => cs 245 __ b(¬_two, cc); 246 __ ldrh(temp1, MemOperand(src, 2, PostIndex)); 247 __ uxtb(temp3, temp1, 8); 248 __ mov(temp3, Operand(temp3, LSL, 16)); 249 __ uxtab(temp3, temp3, temp1); 250 __ str(temp3, MemOperand(dest, 4, PostIndex)); 251 __ bind(¬_two); 252 __ ldrb(temp1, MemOperand(src), ne); 253 __ strh(temp1, MemOperand(dest), ne); 254 __ Pop(pc, r4); 255 } 256 257 CodeDesc desc; 258 masm.GetCode(&desc); 259 260 Assembler::FlushICache(isolate, buffer, actual_size); 261 base::OS::ProtectCode(buffer, actual_size); 262 263 return FUNCTION_CAST<MemCopyUint16Uint8Function>(buffer); 264 #endif 265 } 266 #endif 267 268 UnaryMathFunctionWithIsolate CreateSqrtFunction(Isolate* isolate) { 269 #if defined(USE_SIMULATOR) 270 return nullptr; 271 #else 272 size_t actual_size; 273 byte* buffer = 274 static_cast<byte*>(base::OS::Allocate(1 * KB, &actual_size, true)); 275 if (buffer == nullptr) return nullptr; 276 277 MacroAssembler masm(isolate, buffer, static_cast<int>(actual_size), 278 CodeObjectRequired::kNo); 279 280 __ MovFromFloatParameter(d0); 281 __ vsqrt(d0, d0); 282 __ MovToFloatResult(d0); 283 __ Ret(); 284 285 CodeDesc desc; 286 masm.GetCode(&desc); 287 DCHECK(!RelocInfo::RequiresRelocation(desc)); 288 289 Assembler::FlushICache(isolate, buffer, actual_size); 290 base::OS::ProtectCode(buffer, actual_size); 291 return FUNCTION_CAST<UnaryMathFunctionWithIsolate>(buffer); 292 #endif 293 } 294 295 #undef __ 296 297 298 // ------------------------------------------------------------------------- 299 // Platform-specific RuntimeCallHelper functions. 300 301 void StubRuntimeCallHelper::BeforeCall(MacroAssembler* masm) const { 302 masm->EnterFrame(StackFrame::INTERNAL); 303 DCHECK(!masm->has_frame()); 304 masm->set_has_frame(true); 305 } 306 307 308 void StubRuntimeCallHelper::AfterCall(MacroAssembler* masm) const { 309 masm->LeaveFrame(StackFrame::INTERNAL); 310 DCHECK(masm->has_frame()); 311 masm->set_has_frame(false); 312 } 313 314 315 // ------------------------------------------------------------------------- 316 // Code generators 317 318 #define __ ACCESS_MASM(masm) 319 320 void StringCharLoadGenerator::Generate(MacroAssembler* masm, 321 Register string, 322 Register index, 323 Register result, 324 Label* call_runtime) { 325 Label indirect_string_loaded; 326 __ bind(&indirect_string_loaded); 327 328 // Fetch the instance type of the receiver into result register. 329 __ ldr(result, FieldMemOperand(string, HeapObject::kMapOffset)); 330 __ ldrb(result, FieldMemOperand(result, Map::kInstanceTypeOffset)); 331 332 // We need special handling for indirect strings. 333 Label check_sequential; 334 __ tst(result, Operand(kIsIndirectStringMask)); 335 __ b(eq, &check_sequential); 336 337 // Dispatch on the indirect string shape: slice or cons. 338 Label cons_string, thin_string; 339 __ and_(result, result, Operand(kStringRepresentationMask)); 340 __ cmp(result, Operand(kConsStringTag)); 341 __ b(eq, &cons_string); 342 __ cmp(result, Operand(kThinStringTag)); 343 __ b(eq, &thin_string); 344 345 // Handle slices. 346 __ ldr(result, FieldMemOperand(string, SlicedString::kOffsetOffset)); 347 __ ldr(string, FieldMemOperand(string, SlicedString::kParentOffset)); 348 __ add(index, index, Operand::SmiUntag(result)); 349 __ jmp(&indirect_string_loaded); 350 351 // Handle thin strings. 352 __ bind(&thin_string); 353 __ ldr(string, FieldMemOperand(string, ThinString::kActualOffset)); 354 __ jmp(&indirect_string_loaded); 355 356 // Handle cons strings. 357 // Check whether the right hand side is the empty string (i.e. if 358 // this is really a flat string in a cons string). If that is not 359 // the case we would rather go to the runtime system now to flatten 360 // the string. 361 __ bind(&cons_string); 362 __ ldr(result, FieldMemOperand(string, ConsString::kSecondOffset)); 363 __ CompareRoot(result, Heap::kempty_stringRootIndex); 364 __ b(ne, call_runtime); 365 // Get the first of the two strings and load its instance type. 366 __ ldr(string, FieldMemOperand(string, ConsString::kFirstOffset)); 367 __ jmp(&indirect_string_loaded); 368 369 // Distinguish sequential and external strings. Only these two string 370 // representations can reach here (slices and flat cons strings have been 371 // reduced to the underlying sequential or external string). 372 Label external_string, check_encoding; 373 __ bind(&check_sequential); 374 STATIC_ASSERT(kSeqStringTag == 0); 375 __ tst(result, Operand(kStringRepresentationMask)); 376 __ b(ne, &external_string); 377 378 // Prepare sequential strings 379 STATIC_ASSERT(SeqTwoByteString::kHeaderSize == SeqOneByteString::kHeaderSize); 380 __ add(string, 381 string, 382 Operand(SeqTwoByteString::kHeaderSize - kHeapObjectTag)); 383 __ jmp(&check_encoding); 384 385 // Handle external strings. 386 __ bind(&external_string); 387 if (FLAG_debug_code) { 388 // Assert that we do not have a cons or slice (indirect strings) here. 389 // Sequential strings have already been ruled out. 390 __ tst(result, Operand(kIsIndirectStringMask)); 391 __ Assert(eq, kExternalStringExpectedButNotFound); 392 } 393 // Rule out short external strings. 394 STATIC_ASSERT(kShortExternalStringTag != 0); 395 __ tst(result, Operand(kShortExternalStringMask)); 396 __ b(ne, call_runtime); 397 __ ldr(string, FieldMemOperand(string, ExternalString::kResourceDataOffset)); 398 399 Label one_byte, done; 400 __ bind(&check_encoding); 401 STATIC_ASSERT(kTwoByteStringTag == 0); 402 __ tst(result, Operand(kStringEncodingMask)); 403 __ b(ne, &one_byte); 404 // Two-byte string. 405 __ ldrh(result, MemOperand(string, index, LSL, 1)); 406 __ jmp(&done); 407 __ bind(&one_byte); 408 // One-byte string. 409 __ ldrb(result, MemOperand(string, index)); 410 __ bind(&done); 411 } 412 413 #undef __ 414 415 #ifdef DEBUG 416 // add(r0, pc, Operand(-8)) 417 static const uint32_t kCodeAgePatchFirstInstruction = 0xe24f0008; 418 #endif 419 420 CodeAgingHelper::CodeAgingHelper(Isolate* isolate) { 421 USE(isolate); 422 DCHECK(young_sequence_.length() == kNoCodeAgeSequenceLength); 423 // Since patcher is a large object, allocate it dynamically when needed, 424 // to avoid overloading the stack in stress conditions. 425 // DONT_FLUSH is used because the CodeAgingHelper is initialized early in 426 // the process, before ARM simulator ICache is setup. 427 std::unique_ptr<CodePatcher> patcher( 428 new CodePatcher(isolate, young_sequence_.start(), 429 young_sequence_.length() / Assembler::kInstrSize, 430 CodePatcher::DONT_FLUSH)); 431 PredictableCodeSizeScope scope(patcher->masm(), young_sequence_.length()); 432 patcher->masm()->PushStandardFrame(r1); 433 patcher->masm()->nop(ip.code()); 434 } 435 436 437 #ifdef DEBUG 438 bool CodeAgingHelper::IsOld(byte* candidate) const { 439 return Memory::uint32_at(candidate) == kCodeAgePatchFirstInstruction; 440 } 441 #endif 442 443 444 bool Code::IsYoungSequence(Isolate* isolate, byte* sequence) { 445 bool result = isolate->code_aging_helper()->IsYoung(sequence); 446 DCHECK(result || isolate->code_aging_helper()->IsOld(sequence)); 447 return result; 448 } 449 450 Code::Age Code::GetCodeAge(Isolate* isolate, byte* sequence) { 451 if (IsYoungSequence(isolate, sequence)) return kNoAgeCodeAge; 452 453 Address target_address = Memory::Address_at( 454 sequence + (kNoCodeAgeSequenceLength - Assembler::kInstrSize)); 455 Code* stub = GetCodeFromTargetAddress(target_address); 456 return GetAgeOfCodeAgeStub(stub); 457 } 458 459 void Code::PatchPlatformCodeAge(Isolate* isolate, byte* sequence, 460 Code::Age age) { 461 uint32_t young_length = isolate->code_aging_helper()->young_sequence_length(); 462 if (age == kNoAgeCodeAge) { 463 isolate->code_aging_helper()->CopyYoungSequenceTo(sequence); 464 Assembler::FlushICache(isolate, sequence, young_length); 465 } else { 466 Code* stub = GetCodeAgeStub(isolate, age); 467 CodePatcher patcher(isolate, sequence, 468 young_length / Assembler::kInstrSize); 469 patcher.masm()->add(r0, pc, Operand(-8)); 470 patcher.masm()->ldr(pc, MemOperand(pc, -4)); 471 patcher.masm()->emit_code_stub_address(stub); 472 } 473 } 474 475 } // namespace internal 476 } // namespace v8 477 478 #endif // V8_TARGET_ARCH_ARM 479
