1 // Copyright 2013 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/v8.h" 6 7 #if V8_TARGET_ARCH_ARM64 8 9 #include "src/cpu-profiler.h" 10 #include "src/unicode.h" 11 #include "src/log.h" 12 #include "src/code-stubs.h" 13 #include "src/regexp-stack.h" 14 #include "src/macro-assembler.h" 15 #include "src/regexp-macro-assembler.h" 16 #include "src/arm64/regexp-macro-assembler-arm64.h" 17 18 namespace v8 { 19 namespace internal { 20 21 #ifndef V8_INTERPRETED_REGEXP 22 /* 23 * This assembler uses the following register assignment convention: 24 * - w19 : Used to temporarely store a value before a call to C code. 25 * See CheckNotBackReferenceIgnoreCase. 26 * - x20 : Pointer to the current code object (Code*), 27 * it includes the heap object tag. 28 * - w21 : Current position in input, as negative offset from 29 * the end of the string. Please notice that this is 30 * the byte offset, not the character offset! 31 * - w22 : Currently loaded character. Must be loaded using 32 * LoadCurrentCharacter before using any of the dispatch methods. 33 * - x23 : Points to tip of backtrack stack. 34 * - w24 : Position of the first character minus one: non_position_value. 35 * Used to initialize capture registers. 36 * - x25 : Address at the end of the input string: input_end. 37 * Points to byte after last character in input. 38 * - x26 : Address at the start of the input string: input_start. 39 * - w27 : Where to start in the input string. 40 * - x28 : Output array pointer. 41 * - x29/fp : Frame pointer. Used to access arguments, local variables and 42 * RegExp registers. 43 * - x16/x17 : IP registers, used by assembler. Very volatile. 44 * - csp : Points to tip of C stack. 45 * 46 * - x0-x7 : Used as a cache to store 32 bit capture registers. These 47 * registers need to be retained every time a call to C code 48 * is done. 49 * 50 * The remaining registers are free for computations. 51 * Each call to a public method should retain this convention. 52 * 53 * The stack will have the following structure: 54 * 55 * Location Name Description 56 * (as referred to in 57 * the code) 58 * 59 * - fp[104] isolate Address of the current isolate. 60 * - fp[96] return_address Secondary link/return address 61 * used by an exit frame if this is a 62 * native call. 63 * ^^^ csp when called ^^^ 64 * - fp[88] lr Return from the RegExp code. 65 * - fp[80] r29 Old frame pointer (CalleeSaved). 66 * - fp[0..72] r19-r28 Backup of CalleeSaved registers. 67 * - fp[-8] direct_call 1 => Direct call from JavaScript code. 68 * 0 => Call through the runtime system. 69 * - fp[-16] stack_base High end of the memory area to use as 70 * the backtracking stack. 71 * - fp[-24] output_size Output may fit multiple sets of matches. 72 * - fp[-32] input Handle containing the input string. 73 * - fp[-40] success_counter 74 * ^^^^^^^^^^^^^ From here and downwards we store 32 bit values ^^^^^^^^^^^^^ 75 * - fp[-44] register N Capture registers initialized with 76 * - fp[-48] register N + 1 non_position_value. 77 * ... The first kNumCachedRegisters (N) registers 78 * ... are cached in x0 to x7. 79 * ... Only positions must be stored in the first 80 * - ... num_saved_registers_ registers. 81 * - ... 82 * - register N + num_registers - 1 83 * ^^^^^^^^^ csp ^^^^^^^^^ 84 * 85 * The first num_saved_registers_ registers are initialized to point to 86 * "character -1" in the string (i.e., char_size() bytes before the first 87 * character of the string). The remaining registers start out as garbage. 88 * 89 * The data up to the return address must be placed there by the calling 90 * code and the remaining arguments are passed in registers, e.g. by calling the 91 * code entry as cast to a function with the signature: 92 * int (*match)(String* input, 93 * int start_offset, 94 * Address input_start, 95 * Address input_end, 96 * int* output, 97 * int output_size, 98 * Address stack_base, 99 * bool direct_call = false, 100 * Address secondary_return_address, // Only used by native call. 101 * Isolate* isolate) 102 * The call is performed by NativeRegExpMacroAssembler::Execute() 103 * (in regexp-macro-assembler.cc) via the CALL_GENERATED_REGEXP_CODE macro 104 * in arm64/simulator-arm64.h. 105 * When calling as a non-direct call (i.e., from C++ code), the return address 106 * area is overwritten with the LR register by the RegExp code. When doing a 107 * direct call from generated code, the return address is placed there by 108 * the calling code, as in a normal exit frame. 109 */ 110 111 #define __ ACCESS_MASM(masm_) 112 113 RegExpMacroAssemblerARM64::RegExpMacroAssemblerARM64( 114 Mode mode, 115 int registers_to_save, 116 Zone* zone) 117 : NativeRegExpMacroAssembler(zone), 118 masm_(new MacroAssembler(zone->isolate(), NULL, kRegExpCodeSize)), 119 mode_(mode), 120 num_registers_(registers_to_save), 121 num_saved_registers_(registers_to_save), 122 entry_label_(), 123 start_label_(), 124 success_label_(), 125 backtrack_label_(), 126 exit_label_() { 127 __ SetStackPointer(csp); 128 ASSERT_EQ(0, registers_to_save % 2); 129 // We can cache at most 16 W registers in x0-x7. 130 STATIC_ASSERT(kNumCachedRegisters <= 16); 131 STATIC_ASSERT((kNumCachedRegisters % 2) == 0); 132 __ B(&entry_label_); // We'll write the entry code later. 133 __ Bind(&start_label_); // And then continue from here. 134 } 135 136 137 RegExpMacroAssemblerARM64::~RegExpMacroAssemblerARM64() { 138 delete masm_; 139 // Unuse labels in case we throw away the assembler without calling GetCode. 140 entry_label_.Unuse(); 141 start_label_.Unuse(); 142 success_label_.Unuse(); 143 backtrack_label_.Unuse(); 144 exit_label_.Unuse(); 145 check_preempt_label_.Unuse(); 146 stack_overflow_label_.Unuse(); 147 } 148 149 int RegExpMacroAssemblerARM64::stack_limit_slack() { 150 return RegExpStack::kStackLimitSlack; 151 } 152 153 154 void RegExpMacroAssemblerARM64::AdvanceCurrentPosition(int by) { 155 if (by != 0) { 156 __ Add(current_input_offset(), 157 current_input_offset(), by * char_size()); 158 } 159 } 160 161 162 void RegExpMacroAssemblerARM64::AdvanceRegister(int reg, int by) { 163 ASSERT((reg >= 0) && (reg < num_registers_)); 164 if (by != 0) { 165 Register to_advance; 166 RegisterState register_state = GetRegisterState(reg); 167 switch (register_state) { 168 case STACKED: 169 __ Ldr(w10, register_location(reg)); 170 __ Add(w10, w10, by); 171 __ Str(w10, register_location(reg)); 172 break; 173 case CACHED_LSW: 174 to_advance = GetCachedRegister(reg); 175 __ Add(to_advance, to_advance, by); 176 break; 177 case CACHED_MSW: 178 to_advance = GetCachedRegister(reg); 179 __ Add(to_advance, to_advance, 180 static_cast<int64_t>(by) << kWRegSizeInBits); 181 break; 182 default: 183 UNREACHABLE(); 184 break; 185 } 186 } 187 } 188 189 190 void RegExpMacroAssemblerARM64::Backtrack() { 191 CheckPreemption(); 192 Pop(w10); 193 __ Add(x10, code_pointer(), Operand(w10, UXTW)); 194 __ Br(x10); 195 } 196 197 198 void RegExpMacroAssemblerARM64::Bind(Label* label) { 199 __ Bind(label); 200 } 201 202 203 void RegExpMacroAssemblerARM64::CheckCharacter(uint32_t c, Label* on_equal) { 204 CompareAndBranchOrBacktrack(current_character(), c, eq, on_equal); 205 } 206 207 208 void RegExpMacroAssemblerARM64::CheckCharacterGT(uc16 limit, 209 Label* on_greater) { 210 CompareAndBranchOrBacktrack(current_character(), limit, hi, on_greater); 211 } 212 213 214 void RegExpMacroAssemblerARM64::CheckAtStart(Label* on_at_start) { 215 Label not_at_start; 216 // Did we start the match at the start of the input string? 217 CompareAndBranchOrBacktrack(start_offset(), 0, ne, ¬_at_start); 218 // If we did, are we still at the start of the input string? 219 __ Add(x10, input_end(), Operand(current_input_offset(), SXTW)); 220 __ Cmp(x10, input_start()); 221 BranchOrBacktrack(eq, on_at_start); 222 __ Bind(¬_at_start); 223 } 224 225 226 void RegExpMacroAssemblerARM64::CheckNotAtStart(Label* on_not_at_start) { 227 // Did we start the match at the start of the input string? 228 CompareAndBranchOrBacktrack(start_offset(), 0, ne, on_not_at_start); 229 // If we did, are we still at the start of the input string? 230 __ Add(x10, input_end(), Operand(current_input_offset(), SXTW)); 231 __ Cmp(x10, input_start()); 232 BranchOrBacktrack(ne, on_not_at_start); 233 } 234 235 236 void RegExpMacroAssemblerARM64::CheckCharacterLT(uc16 limit, Label* on_less) { 237 CompareAndBranchOrBacktrack(current_character(), limit, lo, on_less); 238 } 239 240 241 void RegExpMacroAssemblerARM64::CheckCharacters(Vector<const uc16> str, 242 int cp_offset, 243 Label* on_failure, 244 bool check_end_of_string) { 245 // This method is only ever called from the cctests. 246 247 if (check_end_of_string) { 248 // Is last character of required match inside string. 249 CheckPosition(cp_offset + str.length() - 1, on_failure); 250 } 251 252 Register characters_address = x11; 253 254 __ Add(characters_address, 255 input_end(), 256 Operand(current_input_offset(), SXTW)); 257 if (cp_offset != 0) { 258 __ Add(characters_address, characters_address, cp_offset * char_size()); 259 } 260 261 for (int i = 0; i < str.length(); i++) { 262 if (mode_ == ASCII) { 263 __ Ldrb(w10, MemOperand(characters_address, 1, PostIndex)); 264 ASSERT(str[i] <= String::kMaxOneByteCharCode); 265 } else { 266 __ Ldrh(w10, MemOperand(characters_address, 2, PostIndex)); 267 } 268 CompareAndBranchOrBacktrack(w10, str[i], ne, on_failure); 269 } 270 } 271 272 273 void RegExpMacroAssemblerARM64::CheckGreedyLoop(Label* on_equal) { 274 __ Ldr(w10, MemOperand(backtrack_stackpointer())); 275 __ Cmp(current_input_offset(), w10); 276 __ Cset(x11, eq); 277 __ Add(backtrack_stackpointer(), 278 backtrack_stackpointer(), Operand(x11, LSL, kWRegSizeLog2)); 279 BranchOrBacktrack(eq, on_equal); 280 } 281 282 void RegExpMacroAssemblerARM64::CheckNotBackReferenceIgnoreCase( 283 int start_reg, 284 Label* on_no_match) { 285 Label fallthrough; 286 287 Register capture_start_offset = w10; 288 // Save the capture length in a callee-saved register so it will 289 // be preserved if we call a C helper. 290 Register capture_length = w19; 291 ASSERT(kCalleeSaved.IncludesAliasOf(capture_length)); 292 293 // Find length of back-referenced capture. 294 ASSERT((start_reg % 2) == 0); 295 if (start_reg < kNumCachedRegisters) { 296 __ Mov(capture_start_offset.X(), GetCachedRegister(start_reg)); 297 __ Lsr(x11, GetCachedRegister(start_reg), kWRegSizeInBits); 298 } else { 299 __ Ldp(w11, capture_start_offset, capture_location(start_reg, x10)); 300 } 301 __ Sub(capture_length, w11, capture_start_offset); // Length to check. 302 // Succeed on empty capture (including no capture). 303 __ Cbz(capture_length, &fallthrough); 304 305 // Check that there are enough characters left in the input. 306 __ Cmn(capture_length, current_input_offset()); 307 BranchOrBacktrack(gt, on_no_match); 308 309 if (mode_ == ASCII) { 310 Label success; 311 Label fail; 312 Label loop_check; 313 314 Register capture_start_address = x12; 315 Register capture_end_addresss = x13; 316 Register current_position_address = x14; 317 318 __ Add(capture_start_address, 319 input_end(), 320 Operand(capture_start_offset, SXTW)); 321 __ Add(capture_end_addresss, 322 capture_start_address, 323 Operand(capture_length, SXTW)); 324 __ Add(current_position_address, 325 input_end(), 326 Operand(current_input_offset(), SXTW)); 327 328 Label loop; 329 __ Bind(&loop); 330 __ Ldrb(w10, MemOperand(capture_start_address, 1, PostIndex)); 331 __ Ldrb(w11, MemOperand(current_position_address, 1, PostIndex)); 332 __ Cmp(w10, w11); 333 __ B(eq, &loop_check); 334 335 // Mismatch, try case-insensitive match (converting letters to lower-case). 336 __ Orr(w10, w10, 0x20); // Convert capture character to lower-case. 337 __ Orr(w11, w11, 0x20); // Also convert input character. 338 __ Cmp(w11, w10); 339 __ B(ne, &fail); 340 __ Sub(w10, w10, 'a'); 341 __ Cmp(w10, 'z' - 'a'); // Is w10 a lowercase letter? 342 __ B(ls, &loop_check); // In range 'a'-'z'. 343 // Latin-1: Check for values in range [224,254] but not 247. 344 __ Sub(w10, w10, 224 - 'a'); 345 __ Cmp(w10, 254 - 224); 346 __ Ccmp(w10, 247 - 224, ZFlag, ls); // Check for 247. 347 __ B(eq, &fail); // Weren't Latin-1 letters. 348 349 __ Bind(&loop_check); 350 __ Cmp(capture_start_address, capture_end_addresss); 351 __ B(lt, &loop); 352 __ B(&success); 353 354 __ Bind(&fail); 355 BranchOrBacktrack(al, on_no_match); 356 357 __ Bind(&success); 358 // Compute new value of character position after the matched part. 359 __ Sub(current_input_offset().X(), current_position_address, input_end()); 360 if (masm_->emit_debug_code()) { 361 __ Cmp(current_input_offset().X(), Operand(current_input_offset(), SXTW)); 362 __ Ccmp(current_input_offset(), 0, NoFlag, eq); 363 // The current input offset should be <= 0, and fit in a W register. 364 __ Check(le, kOffsetOutOfRange); 365 } 366 } else { 367 ASSERT(mode_ == UC16); 368 int argument_count = 4; 369 370 // The cached registers need to be retained. 371 CPURegList cached_registers(CPURegister::kRegister, kXRegSizeInBits, 0, 7); 372 ASSERT((cached_registers.Count() * 2) == kNumCachedRegisters); 373 __ PushCPURegList(cached_registers); 374 375 // Put arguments into arguments registers. 376 // Parameters are 377 // x0: Address byte_offset1 - Address captured substring's start. 378 // x1: Address byte_offset2 - Address of current character position. 379 // w2: size_t byte_length - length of capture in bytes(!) 380 // x3: Isolate* isolate 381 382 // Address of start of capture. 383 __ Add(x0, input_end(), Operand(capture_start_offset, SXTW)); 384 // Length of capture. 385 __ Mov(w2, capture_length); 386 // Address of current input position. 387 __ Add(x1, input_end(), Operand(current_input_offset(), SXTW)); 388 // Isolate. 389 __ Mov(x3, ExternalReference::isolate_address(isolate())); 390 391 { 392 AllowExternalCallThatCantCauseGC scope(masm_); 393 ExternalReference function = 394 ExternalReference::re_case_insensitive_compare_uc16(isolate()); 395 __ CallCFunction(function, argument_count); 396 } 397 398 // Check if function returned non-zero for success or zero for failure. 399 CompareAndBranchOrBacktrack(x0, 0, eq, on_no_match); 400 // On success, increment position by length of capture. 401 __ Add(current_input_offset(), current_input_offset(), capture_length); 402 // Reset the cached registers. 403 __ PopCPURegList(cached_registers); 404 } 405 406 __ Bind(&fallthrough); 407 } 408 409 void RegExpMacroAssemblerARM64::CheckNotBackReference( 410 int start_reg, 411 Label* on_no_match) { 412 Label fallthrough; 413 414 Register capture_start_address = x12; 415 Register capture_end_address = x13; 416 Register current_position_address = x14; 417 Register capture_length = w15; 418 419 // Find length of back-referenced capture. 420 ASSERT((start_reg % 2) == 0); 421 if (start_reg < kNumCachedRegisters) { 422 __ Mov(x10, GetCachedRegister(start_reg)); 423 __ Lsr(x11, GetCachedRegister(start_reg), kWRegSizeInBits); 424 } else { 425 __ Ldp(w11, w10, capture_location(start_reg, x10)); 426 } 427 __ Sub(capture_length, w11, w10); // Length to check. 428 // Succeed on empty capture (including no capture). 429 __ Cbz(capture_length, &fallthrough); 430 431 // Check that there are enough characters left in the input. 432 __ Cmn(capture_length, current_input_offset()); 433 BranchOrBacktrack(gt, on_no_match); 434 435 // Compute pointers to match string and capture string 436 __ Add(capture_start_address, input_end(), Operand(w10, SXTW)); 437 __ Add(capture_end_address, 438 capture_start_address, 439 Operand(capture_length, SXTW)); 440 __ Add(current_position_address, 441 input_end(), 442 Operand(current_input_offset(), SXTW)); 443 444 Label loop; 445 __ Bind(&loop); 446 if (mode_ == ASCII) { 447 __ Ldrb(w10, MemOperand(capture_start_address, 1, PostIndex)); 448 __ Ldrb(w11, MemOperand(current_position_address, 1, PostIndex)); 449 } else { 450 ASSERT(mode_ == UC16); 451 __ Ldrh(w10, MemOperand(capture_start_address, 2, PostIndex)); 452 __ Ldrh(w11, MemOperand(current_position_address, 2, PostIndex)); 453 } 454 __ Cmp(w10, w11); 455 BranchOrBacktrack(ne, on_no_match); 456 __ Cmp(capture_start_address, capture_end_address); 457 __ B(lt, &loop); 458 459 // Move current character position to position after match. 460 __ Sub(current_input_offset().X(), current_position_address, input_end()); 461 if (masm_->emit_debug_code()) { 462 __ Cmp(current_input_offset().X(), Operand(current_input_offset(), SXTW)); 463 __ Ccmp(current_input_offset(), 0, NoFlag, eq); 464 // The current input offset should be <= 0, and fit in a W register. 465 __ Check(le, kOffsetOutOfRange); 466 } 467 __ Bind(&fallthrough); 468 } 469 470 471 void RegExpMacroAssemblerARM64::CheckNotCharacter(unsigned c, 472 Label* on_not_equal) { 473 CompareAndBranchOrBacktrack(current_character(), c, ne, on_not_equal); 474 } 475 476 477 void RegExpMacroAssemblerARM64::CheckCharacterAfterAnd(uint32_t c, 478 uint32_t mask, 479 Label* on_equal) { 480 __ And(w10, current_character(), mask); 481 CompareAndBranchOrBacktrack(w10, c, eq, on_equal); 482 } 483 484 485 void RegExpMacroAssemblerARM64::CheckNotCharacterAfterAnd(unsigned c, 486 unsigned mask, 487 Label* on_not_equal) { 488 __ And(w10, current_character(), mask); 489 CompareAndBranchOrBacktrack(w10, c, ne, on_not_equal); 490 } 491 492 493 void RegExpMacroAssemblerARM64::CheckNotCharacterAfterMinusAnd( 494 uc16 c, 495 uc16 minus, 496 uc16 mask, 497 Label* on_not_equal) { 498 ASSERT(minus < String::kMaxUtf16CodeUnit); 499 __ Sub(w10, current_character(), minus); 500 __ And(w10, w10, mask); 501 CompareAndBranchOrBacktrack(w10, c, ne, on_not_equal); 502 } 503 504 505 void RegExpMacroAssemblerARM64::CheckCharacterInRange( 506 uc16 from, 507 uc16 to, 508 Label* on_in_range) { 509 __ Sub(w10, current_character(), from); 510 // Unsigned lower-or-same condition. 511 CompareAndBranchOrBacktrack(w10, to - from, ls, on_in_range); 512 } 513 514 515 void RegExpMacroAssemblerARM64::CheckCharacterNotInRange( 516 uc16 from, 517 uc16 to, 518 Label* on_not_in_range) { 519 __ Sub(w10, current_character(), from); 520 // Unsigned higher condition. 521 CompareAndBranchOrBacktrack(w10, to - from, hi, on_not_in_range); 522 } 523 524 525 void RegExpMacroAssemblerARM64::CheckBitInTable( 526 Handle<ByteArray> table, 527 Label* on_bit_set) { 528 __ Mov(x11, Operand(table)); 529 if ((mode_ != ASCII) || (kTableMask != String::kMaxOneByteCharCode)) { 530 __ And(w10, current_character(), kTableMask); 531 __ Add(w10, w10, ByteArray::kHeaderSize - kHeapObjectTag); 532 } else { 533 __ Add(w10, current_character(), ByteArray::kHeaderSize - kHeapObjectTag); 534 } 535 __ Ldrb(w11, MemOperand(x11, w10, UXTW)); 536 CompareAndBranchOrBacktrack(w11, 0, ne, on_bit_set); 537 } 538 539 540 bool RegExpMacroAssemblerARM64::CheckSpecialCharacterClass(uc16 type, 541 Label* on_no_match) { 542 // Range checks (c in min..max) are generally implemented by an unsigned 543 // (c - min) <= (max - min) check 544 switch (type) { 545 case 's': 546 // Match space-characters 547 if (mode_ == ASCII) { 548 // One byte space characters are '\t'..'\r', ' ' and \u00a0. 549 Label success; 550 // Check for ' ' or 0x00a0. 551 __ Cmp(current_character(), ' '); 552 __ Ccmp(current_character(), 0x00a0, ZFlag, ne); 553 __ B(eq, &success); 554 // Check range 0x09..0x0d. 555 __ Sub(w10, current_character(), '\t'); 556 CompareAndBranchOrBacktrack(w10, '\r' - '\t', hi, on_no_match); 557 __ Bind(&success); 558 return true; 559 } 560 return false; 561 case 'S': 562 // The emitted code for generic character classes is good enough. 563 return false; 564 case 'd': 565 // Match ASCII digits ('0'..'9'). 566 __ Sub(w10, current_character(), '0'); 567 CompareAndBranchOrBacktrack(w10, '9' - '0', hi, on_no_match); 568 return true; 569 case 'D': 570 // Match ASCII non-digits. 571 __ Sub(w10, current_character(), '0'); 572 CompareAndBranchOrBacktrack(w10, '9' - '0', ls, on_no_match); 573 return true; 574 case '.': { 575 // Match non-newlines (not 0x0a('\n'), 0x0d('\r'), 0x2028 and 0x2029) 576 // Here we emit the conditional branch only once at the end to make branch 577 // prediction more efficient, even though we could branch out of here 578 // as soon as a character matches. 579 __ Cmp(current_character(), 0x0a); 580 __ Ccmp(current_character(), 0x0d, ZFlag, ne); 581 if (mode_ == UC16) { 582 __ Sub(w10, current_character(), 0x2028); 583 // If the Z flag was set we clear the flags to force a branch. 584 __ Ccmp(w10, 0x2029 - 0x2028, NoFlag, ne); 585 // ls -> !((C==1) && (Z==0)) 586 BranchOrBacktrack(ls, on_no_match); 587 } else { 588 BranchOrBacktrack(eq, on_no_match); 589 } 590 return true; 591 } 592 case 'n': { 593 // Match newlines (0x0a('\n'), 0x0d('\r'), 0x2028 and 0x2029) 594 // We have to check all 4 newline characters before emitting 595 // the conditional branch. 596 __ Cmp(current_character(), 0x0a); 597 __ Ccmp(current_character(), 0x0d, ZFlag, ne); 598 if (mode_ == UC16) { 599 __ Sub(w10, current_character(), 0x2028); 600 // If the Z flag was set we clear the flags to force a fall-through. 601 __ Ccmp(w10, 0x2029 - 0x2028, NoFlag, ne); 602 // hi -> (C==1) && (Z==0) 603 BranchOrBacktrack(hi, on_no_match); 604 } else { 605 BranchOrBacktrack(ne, on_no_match); 606 } 607 return true; 608 } 609 case 'w': { 610 if (mode_ != ASCII) { 611 // Table is 128 entries, so all ASCII characters can be tested. 612 CompareAndBranchOrBacktrack(current_character(), 'z', hi, on_no_match); 613 } 614 ExternalReference map = ExternalReference::re_word_character_map(); 615 __ Mov(x10, map); 616 __ Ldrb(w10, MemOperand(x10, current_character(), UXTW)); 617 CompareAndBranchOrBacktrack(w10, 0, eq, on_no_match); 618 return true; 619 } 620 case 'W': { 621 Label done; 622 if (mode_ != ASCII) { 623 // Table is 128 entries, so all ASCII characters can be tested. 624 __ Cmp(current_character(), 'z'); 625 __ B(hi, &done); 626 } 627 ExternalReference map = ExternalReference::re_word_character_map(); 628 __ Mov(x10, map); 629 __ Ldrb(w10, MemOperand(x10, current_character(), UXTW)); 630 CompareAndBranchOrBacktrack(w10, 0, ne, on_no_match); 631 __ Bind(&done); 632 return true; 633 } 634 case '*': 635 // Match any character. 636 return true; 637 // No custom implementation (yet): s(UC16), S(UC16). 638 default: 639 return false; 640 } 641 } 642 643 644 void RegExpMacroAssemblerARM64::Fail() { 645 __ Mov(w0, FAILURE); 646 __ B(&exit_label_); 647 } 648 649 650 Handle<HeapObject> RegExpMacroAssemblerARM64::GetCode(Handle<String> source) { 651 Label return_w0; 652 // Finalize code - write the entry point code now we know how many 653 // registers we need. 654 655 // Entry code: 656 __ Bind(&entry_label_); 657 658 // Arguments on entry: 659 // x0: String* input 660 // x1: int start_offset 661 // x2: byte* input_start 662 // x3: byte* input_end 663 // x4: int* output array 664 // x5: int output array size 665 // x6: Address stack_base 666 // x7: int direct_call 667 668 // The stack pointer should be csp on entry. 669 // csp[8]: address of the current isolate 670 // csp[0]: secondary link/return address used by native call 671 672 // Tell the system that we have a stack frame. Because the type is MANUAL, no 673 // code is generated. 674 FrameScope scope(masm_, StackFrame::MANUAL); 675 676 // Push registers on the stack, only push the argument registers that we need. 677 CPURegList argument_registers(x0, x5, x6, x7); 678 679 CPURegList registers_to_retain = kCalleeSaved; 680 ASSERT(kCalleeSaved.Count() == 11); 681 registers_to_retain.Combine(lr); 682 683 ASSERT(csp.Is(__ StackPointer())); 684 __ PushCPURegList(registers_to_retain); 685 __ PushCPURegList(argument_registers); 686 687 // Set frame pointer in place. 688 __ Add(frame_pointer(), csp, argument_registers.Count() * kPointerSize); 689 690 // Initialize callee-saved registers. 691 __ Mov(start_offset(), w1); 692 __ Mov(input_start(), x2); 693 __ Mov(input_end(), x3); 694 __ Mov(output_array(), x4); 695 696 // Set the number of registers we will need to allocate, that is: 697 // - success_counter (X register) 698 // - (num_registers_ - kNumCachedRegisters) (W registers) 699 int num_wreg_to_allocate = num_registers_ - kNumCachedRegisters; 700 // Do not allocate registers on the stack if they can all be cached. 701 if (num_wreg_to_allocate < 0) { num_wreg_to_allocate = 0; } 702 // Make room for the success_counter. 703 num_wreg_to_allocate += 2; 704 705 // Make sure the stack alignment will be respected. 706 int alignment = masm_->ActivationFrameAlignment(); 707 ASSERT_EQ(alignment % 16, 0); 708 int align_mask = (alignment / kWRegSize) - 1; 709 num_wreg_to_allocate = (num_wreg_to_allocate + align_mask) & ~align_mask; 710 711 // Check if we have space on the stack. 712 Label stack_limit_hit; 713 Label stack_ok; 714 715 ExternalReference stack_limit = 716 ExternalReference::address_of_stack_limit(isolate()); 717 __ Mov(x10, stack_limit); 718 __ Ldr(x10, MemOperand(x10)); 719 __ Subs(x10, csp, x10); 720 721 // Handle it if the stack pointer is already below the stack limit. 722 __ B(ls, &stack_limit_hit); 723 724 // Check if there is room for the variable number of registers above 725 // the stack limit. 726 __ Cmp(x10, num_wreg_to_allocate * kWRegSize); 727 __ B(hs, &stack_ok); 728 729 // Exit with OutOfMemory exception. There is not enough space on the stack 730 // for our working registers. 731 __ Mov(w0, EXCEPTION); 732 __ B(&return_w0); 733 734 __ Bind(&stack_limit_hit); 735 CallCheckStackGuardState(x10); 736 // If returned value is non-zero, we exit with the returned value as result. 737 __ Cbnz(w0, &return_w0); 738 739 __ Bind(&stack_ok); 740 741 // Allocate space on stack. 742 __ Claim(num_wreg_to_allocate, kWRegSize); 743 744 // Initialize success_counter with 0. 745 __ Str(wzr, MemOperand(frame_pointer(), kSuccessCounter)); 746 747 // Find negative length (offset of start relative to end). 748 __ Sub(x10, input_start(), input_end()); 749 if (masm_->emit_debug_code()) { 750 // Check that the input string length is < 2^30. 751 __ Neg(x11, x10); 752 __ Cmp(x11, (1<<30) - 1); 753 __ Check(ls, kInputStringTooLong); 754 } 755 __ Mov(current_input_offset(), w10); 756 757 // The non-position value is used as a clearing value for the 758 // capture registers, it corresponds to the position of the first character 759 // minus one. 760 __ Sub(non_position_value(), current_input_offset(), char_size()); 761 __ Sub(non_position_value(), non_position_value(), 762 Operand(start_offset(), LSL, (mode_ == UC16) ? 1 : 0)); 763 // We can store this value twice in an X register for initializing 764 // on-stack registers later. 765 __ Orr(twice_non_position_value(), 766 non_position_value().X(), 767 Operand(non_position_value().X(), LSL, kWRegSizeInBits)); 768 769 // Initialize code pointer register. 770 __ Mov(code_pointer(), Operand(masm_->CodeObject())); 771 772 Label load_char_start_regexp, start_regexp; 773 // Load newline if index is at start, previous character otherwise. 774 __ Cbnz(start_offset(), &load_char_start_regexp); 775 __ Mov(current_character(), '\n'); 776 __ B(&start_regexp); 777 778 // Global regexp restarts matching here. 779 __ Bind(&load_char_start_regexp); 780 // Load previous char as initial value of current character register. 781 LoadCurrentCharacterUnchecked(-1, 1); 782 __ Bind(&start_regexp); 783 // Initialize on-stack registers. 784 if (num_saved_registers_ > 0) { 785 ClearRegisters(0, num_saved_registers_ - 1); 786 } 787 788 // Initialize backtrack stack pointer. 789 __ Ldr(backtrack_stackpointer(), MemOperand(frame_pointer(), kStackBase)); 790 791 // Execute 792 __ B(&start_label_); 793 794 if (backtrack_label_.is_linked()) { 795 __ Bind(&backtrack_label_); 796 Backtrack(); 797 } 798 799 if (success_label_.is_linked()) { 800 Register first_capture_start = w15; 801 802 // Save captures when successful. 803 __ Bind(&success_label_); 804 805 if (num_saved_registers_ > 0) { 806 // V8 expects the output to be an int32_t array. 807 Register capture_start = w12; 808 Register capture_end = w13; 809 Register input_length = w14; 810 811 // Copy captures to output. 812 813 // Get string length. 814 __ Sub(x10, input_end(), input_start()); 815 if (masm_->emit_debug_code()) { 816 // Check that the input string length is < 2^30. 817 __ Cmp(x10, (1<<30) - 1); 818 __ Check(ls, kInputStringTooLong); 819 } 820 // input_start has a start_offset offset on entry. We need to include 821 // it when computing the length of the whole string. 822 if (mode_ == UC16) { 823 __ Add(input_length, start_offset(), Operand(w10, LSR, 1)); 824 } else { 825 __ Add(input_length, start_offset(), w10); 826 } 827 828 // Copy the results to the output array from the cached registers first. 829 for (int i = 0; 830 (i < num_saved_registers_) && (i < kNumCachedRegisters); 831 i += 2) { 832 __ Mov(capture_start.X(), GetCachedRegister(i)); 833 __ Lsr(capture_end.X(), capture_start.X(), kWRegSizeInBits); 834 if ((i == 0) && global_with_zero_length_check()) { 835 // Keep capture start for the zero-length check later. 836 __ Mov(first_capture_start, capture_start); 837 } 838 // Offsets need to be relative to the start of the string. 839 if (mode_ == UC16) { 840 __ Add(capture_start, input_length, Operand(capture_start, ASR, 1)); 841 __ Add(capture_end, input_length, Operand(capture_end, ASR, 1)); 842 } else { 843 __ Add(capture_start, input_length, capture_start); 844 __ Add(capture_end, input_length, capture_end); 845 } 846 // The output pointer advances for a possible global match. 847 __ Stp(capture_start, 848 capture_end, 849 MemOperand(output_array(), kPointerSize, PostIndex)); 850 } 851 852 // Only carry on if there are more than kNumCachedRegisters capture 853 // registers. 854 int num_registers_left_on_stack = 855 num_saved_registers_ - kNumCachedRegisters; 856 if (num_registers_left_on_stack > 0) { 857 Register base = x10; 858 // There are always an even number of capture registers. A couple of 859 // registers determine one match with two offsets. 860 ASSERT_EQ(0, num_registers_left_on_stack % 2); 861 __ Add(base, frame_pointer(), kFirstCaptureOnStack); 862 863 // We can unroll the loop here, we should not unroll for less than 2 864 // registers. 865 STATIC_ASSERT(kNumRegistersToUnroll > 2); 866 if (num_registers_left_on_stack <= kNumRegistersToUnroll) { 867 for (int i = 0; i < num_registers_left_on_stack / 2; i++) { 868 __ Ldp(capture_end, 869 capture_start, 870 MemOperand(base, -kPointerSize, PostIndex)); 871 if ((i == 0) && global_with_zero_length_check()) { 872 // Keep capture start for the zero-length check later. 873 __ Mov(first_capture_start, capture_start); 874 } 875 // Offsets need to be relative to the start of the string. 876 if (mode_ == UC16) { 877 __ Add(capture_start, 878 input_length, 879 Operand(capture_start, ASR, 1)); 880 __ Add(capture_end, input_length, Operand(capture_end, ASR, 1)); 881 } else { 882 __ Add(capture_start, input_length, capture_start); 883 __ Add(capture_end, input_length, capture_end); 884 } 885 // The output pointer advances for a possible global match. 886 __ Stp(capture_start, 887 capture_end, 888 MemOperand(output_array(), kPointerSize, PostIndex)); 889 } 890 } else { 891 Label loop, start; 892 __ Mov(x11, num_registers_left_on_stack); 893 894 __ Ldp(capture_end, 895 capture_start, 896 MemOperand(base, -kPointerSize, PostIndex)); 897 if (global_with_zero_length_check()) { 898 __ Mov(first_capture_start, capture_start); 899 } 900 __ B(&start); 901 902 __ Bind(&loop); 903 __ Ldp(capture_end, 904 capture_start, 905 MemOperand(base, -kPointerSize, PostIndex)); 906 __ Bind(&start); 907 if (mode_ == UC16) { 908 __ Add(capture_start, input_length, Operand(capture_start, ASR, 1)); 909 __ Add(capture_end, input_length, Operand(capture_end, ASR, 1)); 910 } else { 911 __ Add(capture_start, input_length, capture_start); 912 __ Add(capture_end, input_length, capture_end); 913 } 914 // The output pointer advances for a possible global match. 915 __ Stp(capture_start, 916 capture_end, 917 MemOperand(output_array(), kPointerSize, PostIndex)); 918 __ Sub(x11, x11, 2); 919 __ Cbnz(x11, &loop); 920 } 921 } 922 } 923 924 if (global()) { 925 Register success_counter = w0; 926 Register output_size = x10; 927 // Restart matching if the regular expression is flagged as global. 928 929 // Increment success counter. 930 __ Ldr(success_counter, MemOperand(frame_pointer(), kSuccessCounter)); 931 __ Add(success_counter, success_counter, 1); 932 __ Str(success_counter, MemOperand(frame_pointer(), kSuccessCounter)); 933 934 // Capture results have been stored, so the number of remaining global 935 // output registers is reduced by the number of stored captures. 936 __ Ldr(output_size, MemOperand(frame_pointer(), kOutputSize)); 937 __ Sub(output_size, output_size, num_saved_registers_); 938 // Check whether we have enough room for another set of capture results. 939 __ Cmp(output_size, num_saved_registers_); 940 __ B(lt, &return_w0); 941 942 // The output pointer is already set to the next field in the output 943 // array. 944 // Update output size on the frame before we restart matching. 945 __ Str(output_size, MemOperand(frame_pointer(), kOutputSize)); 946 947 if (global_with_zero_length_check()) { 948 // Special case for zero-length matches. 949 __ Cmp(current_input_offset(), first_capture_start); 950 // Not a zero-length match, restart. 951 __ B(ne, &load_char_start_regexp); 952 // Offset from the end is zero if we already reached the end. 953 __ Cbz(current_input_offset(), &return_w0); 954 // Advance current position after a zero-length match. 955 __ Add(current_input_offset(), 956 current_input_offset(), 957 Operand((mode_ == UC16) ? 2 : 1)); 958 } 959 960 __ B(&load_char_start_regexp); 961 } else { 962 __ Mov(w0, SUCCESS); 963 } 964 } 965 966 if (exit_label_.is_linked()) { 967 // Exit and return w0 968 __ Bind(&exit_label_); 969 if (global()) { 970 __ Ldr(w0, MemOperand(frame_pointer(), kSuccessCounter)); 971 } 972 } 973 974 __ Bind(&return_w0); 975 976 // Set stack pointer back to first register to retain 977 ASSERT(csp.Is(__ StackPointer())); 978 __ Mov(csp, fp); 979 __ AssertStackConsistency(); 980 981 // Restore registers. 982 __ PopCPURegList(registers_to_retain); 983 984 __ Ret(); 985 986 Label exit_with_exception; 987 // Registers x0 to x7 are used to store the first captures, they need to be 988 // retained over calls to C++ code. 989 CPURegList cached_registers(CPURegister::kRegister, kXRegSizeInBits, 0, 7); 990 ASSERT((cached_registers.Count() * 2) == kNumCachedRegisters); 991 992 if (check_preempt_label_.is_linked()) { 993 __ Bind(&check_preempt_label_); 994 SaveLinkRegister(); 995 // The cached registers need to be retained. 996 __ PushCPURegList(cached_registers); 997 CallCheckStackGuardState(x10); 998 // Returning from the regexp code restores the stack (csp <- fp) 999 // so we don't need to drop the link register from it before exiting. 1000 __ Cbnz(w0, &return_w0); 1001 // Reset the cached registers. 1002 __ PopCPURegList(cached_registers); 1003 RestoreLinkRegister(); 1004 __ Ret(); 1005 } 1006 1007 if (stack_overflow_label_.is_linked()) { 1008 __ Bind(&stack_overflow_label_); 1009 SaveLinkRegister(); 1010 // The cached registers need to be retained. 1011 __ PushCPURegList(cached_registers); 1012 // Call GrowStack(backtrack_stackpointer(), &stack_base) 1013 __ Mov(x2, ExternalReference::isolate_address(isolate())); 1014 __ Add(x1, frame_pointer(), kStackBase); 1015 __ Mov(x0, backtrack_stackpointer()); 1016 ExternalReference grow_stack = 1017 ExternalReference::re_grow_stack(isolate()); 1018 __ CallCFunction(grow_stack, 3); 1019 // If return NULL, we have failed to grow the stack, and 1020 // must exit with a stack-overflow exception. 1021 // Returning from the regexp code restores the stack (csp <- fp) 1022 // so we don't need to drop the link register from it before exiting. 1023 __ Cbz(w0, &exit_with_exception); 1024 // Otherwise use return value as new stack pointer. 1025 __ Mov(backtrack_stackpointer(), x0); 1026 // Reset the cached registers. 1027 __ PopCPURegList(cached_registers); 1028 RestoreLinkRegister(); 1029 __ Ret(); 1030 } 1031 1032 if (exit_with_exception.is_linked()) { 1033 __ Bind(&exit_with_exception); 1034 __ Mov(w0, EXCEPTION); 1035 __ B(&return_w0); 1036 } 1037 1038 CodeDesc code_desc; 1039 masm_->GetCode(&code_desc); 1040 Handle<Code> code = isolate()->factory()->NewCode( 1041 code_desc, Code::ComputeFlags(Code::REGEXP), masm_->CodeObject()); 1042 PROFILE(masm_->isolate(), RegExpCodeCreateEvent(*code, *source)); 1043 return Handle<HeapObject>::cast(code); 1044 } 1045 1046 1047 void RegExpMacroAssemblerARM64::GoTo(Label* to) { 1048 BranchOrBacktrack(al, to); 1049 } 1050 1051 void RegExpMacroAssemblerARM64::IfRegisterGE(int reg, int comparand, 1052 Label* if_ge) { 1053 Register to_compare = GetRegister(reg, w10); 1054 CompareAndBranchOrBacktrack(to_compare, comparand, ge, if_ge); 1055 } 1056 1057 1058 void RegExpMacroAssemblerARM64::IfRegisterLT(int reg, int comparand, 1059 Label* if_lt) { 1060 Register to_compare = GetRegister(reg, w10); 1061 CompareAndBranchOrBacktrack(to_compare, comparand, lt, if_lt); 1062 } 1063 1064 1065 void RegExpMacroAssemblerARM64::IfRegisterEqPos(int reg, Label* if_eq) { 1066 Register to_compare = GetRegister(reg, w10); 1067 __ Cmp(to_compare, current_input_offset()); 1068 BranchOrBacktrack(eq, if_eq); 1069 } 1070 1071 RegExpMacroAssembler::IrregexpImplementation 1072 RegExpMacroAssemblerARM64::Implementation() { 1073 return kARM64Implementation; 1074 } 1075 1076 1077 void RegExpMacroAssemblerARM64::LoadCurrentCharacter(int cp_offset, 1078 Label* on_end_of_input, 1079 bool check_bounds, 1080 int characters) { 1081 // TODO(pielan): Make sure long strings are caught before this, and not 1082 // just asserted in debug mode. 1083 ASSERT(cp_offset >= -1); // ^ and \b can look behind one character. 1084 // Be sane! (And ensure that an int32_t can be used to index the string) 1085 ASSERT(cp_offset < (1<<30)); 1086 if (check_bounds) { 1087 CheckPosition(cp_offset + characters - 1, on_end_of_input); 1088 } 1089 LoadCurrentCharacterUnchecked(cp_offset, characters); 1090 } 1091 1092 1093 void RegExpMacroAssemblerARM64::PopCurrentPosition() { 1094 Pop(current_input_offset()); 1095 } 1096 1097 1098 void RegExpMacroAssemblerARM64::PopRegister(int register_index) { 1099 Pop(w10); 1100 StoreRegister(register_index, w10); 1101 } 1102 1103 1104 void RegExpMacroAssemblerARM64::PushBacktrack(Label* label) { 1105 if (label->is_bound()) { 1106 int target = label->pos(); 1107 __ Mov(w10, target + Code::kHeaderSize - kHeapObjectTag); 1108 } else { 1109 __ Adr(x10, label, MacroAssembler::kAdrFar); 1110 __ Sub(x10, x10, code_pointer()); 1111 if (masm_->emit_debug_code()) { 1112 __ Cmp(x10, kWRegMask); 1113 // The code offset has to fit in a W register. 1114 __ Check(ls, kOffsetOutOfRange); 1115 } 1116 } 1117 Push(w10); 1118 CheckStackLimit(); 1119 } 1120 1121 1122 void RegExpMacroAssemblerARM64::PushCurrentPosition() { 1123 Push(current_input_offset()); 1124 } 1125 1126 1127 void RegExpMacroAssemblerARM64::PushRegister(int register_index, 1128 StackCheckFlag check_stack_limit) { 1129 Register to_push = GetRegister(register_index, w10); 1130 Push(to_push); 1131 if (check_stack_limit) CheckStackLimit(); 1132 } 1133 1134 1135 void RegExpMacroAssemblerARM64::ReadCurrentPositionFromRegister(int reg) { 1136 Register cached_register; 1137 RegisterState register_state = GetRegisterState(reg); 1138 switch (register_state) { 1139 case STACKED: 1140 __ Ldr(current_input_offset(), register_location(reg)); 1141 break; 1142 case CACHED_LSW: 1143 cached_register = GetCachedRegister(reg); 1144 __ Mov(current_input_offset(), cached_register.W()); 1145 break; 1146 case CACHED_MSW: 1147 cached_register = GetCachedRegister(reg); 1148 __ Lsr(current_input_offset().X(), cached_register, kWRegSizeInBits); 1149 break; 1150 default: 1151 UNREACHABLE(); 1152 break; 1153 } 1154 } 1155 1156 1157 void RegExpMacroAssemblerARM64::ReadStackPointerFromRegister(int reg) { 1158 Register read_from = GetRegister(reg, w10); 1159 __ Ldr(x11, MemOperand(frame_pointer(), kStackBase)); 1160 __ Add(backtrack_stackpointer(), x11, Operand(read_from, SXTW)); 1161 } 1162 1163 1164 void RegExpMacroAssemblerARM64::SetCurrentPositionFromEnd(int by) { 1165 Label after_position; 1166 __ Cmp(current_input_offset(), -by * char_size()); 1167 __ B(ge, &after_position); 1168 __ Mov(current_input_offset(), -by * char_size()); 1169 // On RegExp code entry (where this operation is used), the character before 1170 // the current position is expected to be already loaded. 1171 // We have advanced the position, so it's safe to read backwards. 1172 LoadCurrentCharacterUnchecked(-1, 1); 1173 __ Bind(&after_position); 1174 } 1175 1176 1177 void RegExpMacroAssemblerARM64::SetRegister(int register_index, int to) { 1178 ASSERT(register_index >= num_saved_registers_); // Reserved for positions! 1179 Register set_to = wzr; 1180 if (to != 0) { 1181 set_to = w10; 1182 __ Mov(set_to, to); 1183 } 1184 StoreRegister(register_index, set_to); 1185 } 1186 1187 1188 bool RegExpMacroAssemblerARM64::Succeed() { 1189 __ B(&success_label_); 1190 return global(); 1191 } 1192 1193 1194 void RegExpMacroAssemblerARM64::WriteCurrentPositionToRegister(int reg, 1195 int cp_offset) { 1196 Register position = current_input_offset(); 1197 if (cp_offset != 0) { 1198 position = w10; 1199 __ Add(position, current_input_offset(), cp_offset * char_size()); 1200 } 1201 StoreRegister(reg, position); 1202 } 1203 1204 1205 void RegExpMacroAssemblerARM64::ClearRegisters(int reg_from, int reg_to) { 1206 ASSERT(reg_from <= reg_to); 1207 int num_registers = reg_to - reg_from + 1; 1208 1209 // If the first capture register is cached in a hardware register but not 1210 // aligned on a 64-bit one, we need to clear the first one specifically. 1211 if ((reg_from < kNumCachedRegisters) && ((reg_from % 2) != 0)) { 1212 StoreRegister(reg_from, non_position_value()); 1213 num_registers--; 1214 reg_from++; 1215 } 1216 1217 // Clear cached registers in pairs as far as possible. 1218 while ((num_registers >= 2) && (reg_from < kNumCachedRegisters)) { 1219 ASSERT(GetRegisterState(reg_from) == CACHED_LSW); 1220 __ Mov(GetCachedRegister(reg_from), twice_non_position_value()); 1221 reg_from += 2; 1222 num_registers -= 2; 1223 } 1224 1225 if ((num_registers % 2) == 1) { 1226 StoreRegister(reg_from, non_position_value()); 1227 num_registers--; 1228 reg_from++; 1229 } 1230 1231 if (num_registers > 0) { 1232 // If there are some remaining registers, they are stored on the stack. 1233 ASSERT(reg_from >= kNumCachedRegisters); 1234 1235 // Move down the indexes of the registers on stack to get the correct offset 1236 // in memory. 1237 reg_from -= kNumCachedRegisters; 1238 reg_to -= kNumCachedRegisters; 1239 // We should not unroll the loop for less than 2 registers. 1240 STATIC_ASSERT(kNumRegistersToUnroll > 2); 1241 // We position the base pointer to (reg_from + 1). 1242 int base_offset = kFirstRegisterOnStack - 1243 kWRegSize - (kWRegSize * reg_from); 1244 if (num_registers > kNumRegistersToUnroll) { 1245 Register base = x10; 1246 __ Add(base, frame_pointer(), base_offset); 1247 1248 Label loop; 1249 __ Mov(x11, num_registers); 1250 __ Bind(&loop); 1251 __ Str(twice_non_position_value(), 1252 MemOperand(base, -kPointerSize, PostIndex)); 1253 __ Sub(x11, x11, 2); 1254 __ Cbnz(x11, &loop); 1255 } else { 1256 for (int i = reg_from; i <= reg_to; i += 2) { 1257 __ Str(twice_non_position_value(), 1258 MemOperand(frame_pointer(), base_offset)); 1259 base_offset -= kWRegSize * 2; 1260 } 1261 } 1262 } 1263 } 1264 1265 1266 void RegExpMacroAssemblerARM64::WriteStackPointerToRegister(int reg) { 1267 __ Ldr(x10, MemOperand(frame_pointer(), kStackBase)); 1268 __ Sub(x10, backtrack_stackpointer(), x10); 1269 if (masm_->emit_debug_code()) { 1270 __ Cmp(x10, Operand(w10, SXTW)); 1271 // The stack offset needs to fit in a W register. 1272 __ Check(eq, kOffsetOutOfRange); 1273 } 1274 StoreRegister(reg, w10); 1275 } 1276 1277 1278 // Helper function for reading a value out of a stack frame. 1279 template <typename T> 1280 static T& frame_entry(Address re_frame, int frame_offset) { 1281 return *reinterpret_cast<T*>(re_frame + frame_offset); 1282 } 1283 1284 1285 int RegExpMacroAssemblerARM64::CheckStackGuardState(Address* return_address, 1286 Code* re_code, 1287 Address re_frame, 1288 int start_offset, 1289 const byte** input_start, 1290 const byte** input_end) { 1291 Isolate* isolate = frame_entry<Isolate*>(re_frame, kIsolate); 1292 StackLimitCheck check(isolate); 1293 if (check.JsHasOverflowed()) { 1294 isolate->StackOverflow(); 1295 return EXCEPTION; 1296 } 1297 1298 // If not real stack overflow the stack guard was used to interrupt 1299 // execution for another purpose. 1300 1301 // If this is a direct call from JavaScript retry the RegExp forcing the call 1302 // through the runtime system. Currently the direct call cannot handle a GC. 1303 if (frame_entry<int>(re_frame, kDirectCall) == 1) { 1304 return RETRY; 1305 } 1306 1307 // Prepare for possible GC. 1308 HandleScope handles(isolate); 1309 Handle<Code> code_handle(re_code); 1310 1311 Handle<String> subject(frame_entry<String*>(re_frame, kInput)); 1312 1313 // Current string. 1314 bool is_ascii = subject->IsOneByteRepresentationUnderneath(); 1315 1316 ASSERT(re_code->instruction_start() <= *return_address); 1317 ASSERT(*return_address <= 1318 re_code->instruction_start() + re_code->instruction_size()); 1319 1320 Object* result = isolate->stack_guard()->HandleInterrupts(); 1321 1322 if (*code_handle != re_code) { // Return address no longer valid 1323 int delta = code_handle->address() - re_code->address(); 1324 // Overwrite the return address on the stack. 1325 *return_address += delta; 1326 } 1327 1328 if (result->IsException()) { 1329 return EXCEPTION; 1330 } 1331 1332 Handle<String> subject_tmp = subject; 1333 int slice_offset = 0; 1334 1335 // Extract the underlying string and the slice offset. 1336 if (StringShape(*subject_tmp).IsCons()) { 1337 subject_tmp = Handle<String>(ConsString::cast(*subject_tmp)->first()); 1338 } else if (StringShape(*subject_tmp).IsSliced()) { 1339 SlicedString* slice = SlicedString::cast(*subject_tmp); 1340 subject_tmp = Handle<String>(slice->parent()); 1341 slice_offset = slice->offset(); 1342 } 1343 1344 // String might have changed. 1345 if (subject_tmp->IsOneByteRepresentation() != is_ascii) { 1346 // If we changed between an ASCII and an UC16 string, the specialized 1347 // code cannot be used, and we need to restart regexp matching from 1348 // scratch (including, potentially, compiling a new version of the code). 1349 return RETRY; 1350 } 1351 1352 // Otherwise, the content of the string might have moved. It must still 1353 // be a sequential or external string with the same content. 1354 // Update the start and end pointers in the stack frame to the current 1355 // location (whether it has actually moved or not). 1356 ASSERT(StringShape(*subject_tmp).IsSequential() || 1357 StringShape(*subject_tmp).IsExternal()); 1358 1359 // The original start address of the characters to match. 1360 const byte* start_address = *input_start; 1361 1362 // Find the current start address of the same character at the current string 1363 // position. 1364 const byte* new_address = StringCharacterPosition(*subject_tmp, 1365 start_offset + slice_offset); 1366 1367 if (start_address != new_address) { 1368 // If there is a difference, update the object pointer and start and end 1369 // addresses in the RegExp stack frame to match the new value. 1370 const byte* end_address = *input_end; 1371 int byte_length = static_cast<int>(end_address - start_address); 1372 frame_entry<const String*>(re_frame, kInput) = *subject; 1373 *input_start = new_address; 1374 *input_end = new_address + byte_length; 1375 } else if (frame_entry<const String*>(re_frame, kInput) != *subject) { 1376 // Subject string might have been a ConsString that underwent 1377 // short-circuiting during GC. That will not change start_address but 1378 // will change pointer inside the subject handle. 1379 frame_entry<const String*>(re_frame, kInput) = *subject; 1380 } 1381 1382 return 0; 1383 } 1384 1385 1386 void RegExpMacroAssemblerARM64::CheckPosition(int cp_offset, 1387 Label* on_outside_input) { 1388 CompareAndBranchOrBacktrack(current_input_offset(), 1389 -cp_offset * char_size(), 1390 ge, 1391 on_outside_input); 1392 } 1393 1394 1395 bool RegExpMacroAssemblerARM64::CanReadUnaligned() { 1396 // TODO(pielan): See whether or not we should disable unaligned accesses. 1397 return !slow_safe(); 1398 } 1399 1400 1401 // Private methods: 1402 1403 void RegExpMacroAssemblerARM64::CallCheckStackGuardState(Register scratch) { 1404 // Allocate space on the stack to store the return address. The 1405 // CheckStackGuardState C++ function will override it if the code 1406 // moved. Allocate extra space for 2 arguments passed by pointers. 1407 // AAPCS64 requires the stack to be 16 byte aligned. 1408 int alignment = masm_->ActivationFrameAlignment(); 1409 ASSERT_EQ(alignment % 16, 0); 1410 int align_mask = (alignment / kXRegSize) - 1; 1411 int xreg_to_claim = (3 + align_mask) & ~align_mask; 1412 1413 ASSERT(csp.Is(__ StackPointer())); 1414 __ Claim(xreg_to_claim); 1415 1416 // CheckStackGuardState needs the end and start addresses of the input string. 1417 __ Poke(input_end(), 2 * kPointerSize); 1418 __ Add(x5, csp, 2 * kPointerSize); 1419 __ Poke(input_start(), kPointerSize); 1420 __ Add(x4, csp, kPointerSize); 1421 1422 __ Mov(w3, start_offset()); 1423 // RegExp code frame pointer. 1424 __ Mov(x2, frame_pointer()); 1425 // Code* of self. 1426 __ Mov(x1, Operand(masm_->CodeObject())); 1427 1428 // We need to pass a pointer to the return address as first argument. 1429 // The DirectCEntry stub will place the return address on the stack before 1430 // calling so the stack pointer will point to it. 1431 __ Mov(x0, csp); 1432 1433 ExternalReference check_stack_guard_state = 1434 ExternalReference::re_check_stack_guard_state(isolate()); 1435 __ Mov(scratch, check_stack_guard_state); 1436 DirectCEntryStub stub(isolate()); 1437 stub.GenerateCall(masm_, scratch); 1438 1439 // The input string may have been moved in memory, we need to reload it. 1440 __ Peek(input_start(), kPointerSize); 1441 __ Peek(input_end(), 2 * kPointerSize); 1442 1443 ASSERT(csp.Is(__ StackPointer())); 1444 __ Drop(xreg_to_claim); 1445 1446 // Reload the Code pointer. 1447 __ Mov(code_pointer(), Operand(masm_->CodeObject())); 1448 } 1449 1450 void RegExpMacroAssemblerARM64::BranchOrBacktrack(Condition condition, 1451 Label* to) { 1452 if (condition == al) { // Unconditional. 1453 if (to == NULL) { 1454 Backtrack(); 1455 return; 1456 } 1457 __ B(to); 1458 return; 1459 } 1460 if (to == NULL) { 1461 to = &backtrack_label_; 1462 } 1463 // TODO(ulan): do direct jump when jump distance is known and fits in imm19. 1464 Condition inverted_condition = NegateCondition(condition); 1465 Label no_branch; 1466 __ B(inverted_condition, &no_branch); 1467 __ B(to); 1468 __ Bind(&no_branch); 1469 } 1470 1471 void RegExpMacroAssemblerARM64::CompareAndBranchOrBacktrack(Register reg, 1472 int immediate, 1473 Condition condition, 1474 Label* to) { 1475 if ((immediate == 0) && ((condition == eq) || (condition == ne))) { 1476 if (to == NULL) { 1477 to = &backtrack_label_; 1478 } 1479 // TODO(ulan): do direct jump when jump distance is known and fits in imm19. 1480 Label no_branch; 1481 if (condition == eq) { 1482 __ Cbnz(reg, &no_branch); 1483 } else { 1484 __ Cbz(reg, &no_branch); 1485 } 1486 __ B(to); 1487 __ Bind(&no_branch); 1488 } else { 1489 __ Cmp(reg, immediate); 1490 BranchOrBacktrack(condition, to); 1491 } 1492 } 1493 1494 1495 void RegExpMacroAssemblerARM64::CheckPreemption() { 1496 // Check for preemption. 1497 ExternalReference stack_limit = 1498 ExternalReference::address_of_stack_limit(isolate()); 1499 __ Mov(x10, stack_limit); 1500 __ Ldr(x10, MemOperand(x10)); 1501 ASSERT(csp.Is(__ StackPointer())); 1502 __ Cmp(csp, x10); 1503 CallIf(&check_preempt_label_, ls); 1504 } 1505 1506 1507 void RegExpMacroAssemblerARM64::CheckStackLimit() { 1508 ExternalReference stack_limit = 1509 ExternalReference::address_of_regexp_stack_limit(isolate()); 1510 __ Mov(x10, stack_limit); 1511 __ Ldr(x10, MemOperand(x10)); 1512 __ Cmp(backtrack_stackpointer(), x10); 1513 CallIf(&stack_overflow_label_, ls); 1514 } 1515 1516 1517 void RegExpMacroAssemblerARM64::Push(Register source) { 1518 ASSERT(source.Is32Bits()); 1519 ASSERT(!source.is(backtrack_stackpointer())); 1520 __ Str(source, 1521 MemOperand(backtrack_stackpointer(), 1522 -static_cast<int>(kWRegSize), 1523 PreIndex)); 1524 } 1525 1526 1527 void RegExpMacroAssemblerARM64::Pop(Register target) { 1528 ASSERT(target.Is32Bits()); 1529 ASSERT(!target.is(backtrack_stackpointer())); 1530 __ Ldr(target, 1531 MemOperand(backtrack_stackpointer(), kWRegSize, PostIndex)); 1532 } 1533 1534 1535 Register RegExpMacroAssemblerARM64::GetCachedRegister(int register_index) { 1536 ASSERT(register_index < kNumCachedRegisters); 1537 return Register::Create(register_index / 2, kXRegSizeInBits); 1538 } 1539 1540 1541 Register RegExpMacroAssemblerARM64::GetRegister(int register_index, 1542 Register maybe_result) { 1543 ASSERT(maybe_result.Is32Bits()); 1544 ASSERT(register_index >= 0); 1545 if (num_registers_ <= register_index) { 1546 num_registers_ = register_index + 1; 1547 } 1548 Register result; 1549 RegisterState register_state = GetRegisterState(register_index); 1550 switch (register_state) { 1551 case STACKED: 1552 __ Ldr(maybe_result, register_location(register_index)); 1553 result = maybe_result; 1554 break; 1555 case CACHED_LSW: 1556 result = GetCachedRegister(register_index).W(); 1557 break; 1558 case CACHED_MSW: 1559 __ Lsr(maybe_result.X(), GetCachedRegister(register_index), 1560 kWRegSizeInBits); 1561 result = maybe_result; 1562 break; 1563 default: 1564 UNREACHABLE(); 1565 break; 1566 } 1567 ASSERT(result.Is32Bits()); 1568 return result; 1569 } 1570 1571 1572 void RegExpMacroAssemblerARM64::StoreRegister(int register_index, 1573 Register source) { 1574 ASSERT(source.Is32Bits()); 1575 ASSERT(register_index >= 0); 1576 if (num_registers_ <= register_index) { 1577 num_registers_ = register_index + 1; 1578 } 1579 1580 Register cached_register; 1581 RegisterState register_state = GetRegisterState(register_index); 1582 switch (register_state) { 1583 case STACKED: 1584 __ Str(source, register_location(register_index)); 1585 break; 1586 case CACHED_LSW: 1587 cached_register = GetCachedRegister(register_index); 1588 if (!source.Is(cached_register.W())) { 1589 __ Bfi(cached_register, source.X(), 0, kWRegSizeInBits); 1590 } 1591 break; 1592 case CACHED_MSW: 1593 cached_register = GetCachedRegister(register_index); 1594 __ Bfi(cached_register, source.X(), kWRegSizeInBits, kWRegSizeInBits); 1595 break; 1596 default: 1597 UNREACHABLE(); 1598 break; 1599 } 1600 } 1601 1602 1603 void RegExpMacroAssemblerARM64::CallIf(Label* to, Condition condition) { 1604 Label skip_call; 1605 if (condition != al) __ B(&skip_call, NegateCondition(condition)); 1606 __ Bl(to); 1607 __ Bind(&skip_call); 1608 } 1609 1610 1611 void RegExpMacroAssemblerARM64::RestoreLinkRegister() { 1612 ASSERT(csp.Is(__ StackPointer())); 1613 __ Pop(lr, xzr); 1614 __ Add(lr, lr, Operand(masm_->CodeObject())); 1615 } 1616 1617 1618 void RegExpMacroAssemblerARM64::SaveLinkRegister() { 1619 ASSERT(csp.Is(__ StackPointer())); 1620 __ Sub(lr, lr, Operand(masm_->CodeObject())); 1621 __ Push(xzr, lr); 1622 } 1623 1624 1625 MemOperand RegExpMacroAssemblerARM64::register_location(int register_index) { 1626 ASSERT(register_index < (1<<30)); 1627 ASSERT(register_index >= kNumCachedRegisters); 1628 if (num_registers_ <= register_index) { 1629 num_registers_ = register_index + 1; 1630 } 1631 register_index -= kNumCachedRegisters; 1632 int offset = kFirstRegisterOnStack - register_index * kWRegSize; 1633 return MemOperand(frame_pointer(), offset); 1634 } 1635 1636 MemOperand RegExpMacroAssemblerARM64::capture_location(int register_index, 1637 Register scratch) { 1638 ASSERT(register_index < (1<<30)); 1639 ASSERT(register_index < num_saved_registers_); 1640 ASSERT(register_index >= kNumCachedRegisters); 1641 ASSERT_EQ(register_index % 2, 0); 1642 register_index -= kNumCachedRegisters; 1643 int offset = kFirstCaptureOnStack - register_index * kWRegSize; 1644 // capture_location is used with Stp instructions to load/store 2 registers. 1645 // The immediate field in the encoding is limited to 7 bits (signed). 1646 if (is_int7(offset)) { 1647 return MemOperand(frame_pointer(), offset); 1648 } else { 1649 __ Add(scratch, frame_pointer(), offset); 1650 return MemOperand(scratch); 1651 } 1652 } 1653 1654 void RegExpMacroAssemblerARM64::LoadCurrentCharacterUnchecked(int cp_offset, 1655 int characters) { 1656 Register offset = current_input_offset(); 1657 1658 // The ldr, str, ldrh, strh instructions can do unaligned accesses, if the CPU 1659 // and the operating system running on the target allow it. 1660 // If unaligned load/stores are not supported then this function must only 1661 // be used to load a single character at a time. 1662 1663 // ARMv8 supports unaligned accesses but V8 or the kernel can decide to 1664 // disable it. 1665 // TODO(pielan): See whether or not we should disable unaligned accesses. 1666 if (!CanReadUnaligned()) { 1667 ASSERT(characters == 1); 1668 } 1669 1670 if (cp_offset != 0) { 1671 if (masm_->emit_debug_code()) { 1672 __ Mov(x10, cp_offset * char_size()); 1673 __ Add(x10, x10, Operand(current_input_offset(), SXTW)); 1674 __ Cmp(x10, Operand(w10, SXTW)); 1675 // The offset needs to fit in a W register. 1676 __ Check(eq, kOffsetOutOfRange); 1677 } else { 1678 __ Add(w10, current_input_offset(), cp_offset * char_size()); 1679 } 1680 offset = w10; 1681 } 1682 1683 if (mode_ == ASCII) { 1684 if (characters == 4) { 1685 __ Ldr(current_character(), MemOperand(input_end(), offset, SXTW)); 1686 } else if (characters == 2) { 1687 __ Ldrh(current_character(), MemOperand(input_end(), offset, SXTW)); 1688 } else { 1689 ASSERT(characters == 1); 1690 __ Ldrb(current_character(), MemOperand(input_end(), offset, SXTW)); 1691 } 1692 } else { 1693 ASSERT(mode_ == UC16); 1694 if (characters == 2) { 1695 __ Ldr(current_character(), MemOperand(input_end(), offset, SXTW)); 1696 } else { 1697 ASSERT(characters == 1); 1698 __ Ldrh(current_character(), MemOperand(input_end(), offset, SXTW)); 1699 } 1700 } 1701 } 1702 1703 #endif // V8_INTERPRETED_REGEXP 1704 1705 }} // namespace v8::internal 1706 1707 #endif // V8_TARGET_ARCH_ARM64 1708
