1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "codegen_arm64.h" 18 19 #include "arch/arm64/instruction_set_features_arm64.h" 20 #include "arm64_lir.h" 21 #include "base/logging.h" 22 #include "dex/compiler_ir.h" 23 #include "dex/quick/mir_to_lir-inl.h" 24 #include "driver/compiler_driver.h" 25 26 namespace art { 27 28 // The macros below are exclusively used in the encoding map. 29 30 // Most generic way of providing two variants for one instructions. 31 #define CUSTOM_VARIANTS(variant1, variant2) variant1, variant2 32 33 // Used for instructions which do not have a wide variant. 34 #define NO_VARIANTS(variant) \ 35 CUSTOM_VARIANTS(variant, 0) 36 37 // Used for instructions which have a wide variant with the sf bit set to 1. 38 #define SF_VARIANTS(sf0_skeleton) \ 39 CUSTOM_VARIANTS(sf0_skeleton, (sf0_skeleton | 0x80000000)) 40 41 // Used for instructions which have a wide variant with the size bits set to either x0 or x1. 42 #define SIZE_VARIANTS(sizex0_skeleton) \ 43 CUSTOM_VARIANTS(sizex0_skeleton, (sizex0_skeleton | 0x40000000)) 44 45 // Used for instructions which have a wide variant with the sf and n bits set to 1. 46 #define SF_N_VARIANTS(sf0_n0_skeleton) \ 47 CUSTOM_VARIANTS(sf0_n0_skeleton, (sf0_n0_skeleton | 0x80400000)) 48 49 // Used for FP instructions which have a single and double precision variants, with he type bits set 50 // to either 00 or 01. 51 #define FLOAT_VARIANTS(type00_skeleton) \ 52 CUSTOM_VARIANTS(type00_skeleton, (type00_skeleton | 0x00400000)) 53 54 /* 55 * opcode: A64Opcode enum 56 * variants: instruction skeletons supplied via CUSTOM_VARIANTS or derived macros. 57 * a{n}k: key to applying argument {n} \ 58 * a{n}s: argument {n} start bit position | n = 0, 1, 2, 3 59 * a{n}e: argument {n} end bit position / 60 * flags: instruction attributes (used in optimization) 61 * name: mnemonic name 62 * fmt: for pretty-printing 63 * fixup: used for second-pass fixes (e.g. adresses fixups in branch instructions). 64 */ 65 #define ENCODING_MAP(opcode, variants, a0k, a0s, a0e, a1k, a1s, a1e, a2k, a2s, a2e, \ 66 a3k, a3s, a3e, flags, name, fmt, fixup) \ 67 {variants, {{a0k, a0s, a0e}, {a1k, a1s, a1e}, {a2k, a2s, a2e}, \ 68 {a3k, a3s, a3e}}, opcode, flags, name, fmt, 4, fixup} 69 70 /* Instruction dump string format keys: !pf, where "!" is the start 71 * of the key, "p" is which numeric operand to use and "f" is the 72 * print format. 73 * 74 * [p]ositions: 75 * 0 -> operands[0] (dest) 76 * 1 -> operands[1] (src1) 77 * 2 -> operands[2] (src2) 78 * 3 -> operands[3] (extra) 79 * 80 * [f]ormats: 81 * d -> decimal 82 * D -> decimal*4 or decimal*8 depending on the instruction width 83 * E -> decimal*4 84 * F -> decimal*2 85 * G -> ", lsl #2" or ", lsl #3" depending on the instruction width 86 * c -> branch condition (eq, ne, etc.) 87 * t -> pc-relative target 88 * p -> pc-relative address 89 * s -> single precision floating point register 90 * S -> double precision floating point register 91 * f -> single or double precision register (depending on instruction width) 92 * I -> 8-bit immediate floating point number 93 * l -> logical immediate 94 * M -> 16-bit shift expression ("" or ", lsl #16" or ", lsl #32"...) 95 * B -> dmb option string (sy, st, ish, ishst, nsh, hshst) 96 * H -> operand shift 97 * h -> 6-bit shift immediate 98 * T -> register shift (either ", lsl #0" or ", lsl #12") 99 * e -> register extend (e.g. uxtb #1) 100 * o -> register shift (e.g. lsl #1) for Word registers 101 * w -> word (32-bit) register wn, or wzr 102 * W -> word (32-bit) register wn, or wsp 103 * x -> extended (64-bit) register xn, or xzr 104 * X -> extended (64-bit) register xn, or sp 105 * r -> register with same width as instruction, r31 -> wzr, xzr 106 * R -> register with same width as instruction, r31 -> wsp, sp 107 * 108 * [!] escape. To insert "!", use "!!" 109 */ 110 /* NOTE: must be kept in sync with enum A64Opcode from arm64_lir.h */ 111 const A64EncodingMap Arm64Mir2Lir::EncodingMap[kA64Last] = { 112 ENCODING_MAP(WIDE(kA64Adc3rrr), SF_VARIANTS(0x1a000000), 113 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 114 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | USES_CCODES, 115 "adc", "!0r, !1r, !2r", kFixupNone), 116 ENCODING_MAP(WIDE(kA64Add4RRdT), SF_VARIANTS(0x11000000), 117 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10, 118 kFmtBitBlt, 23, 22, IS_QUAD_OP | REG_DEF0_USE1, 119 "add", "!0R, !1R, #!2d!3T", kFixupNone), 120 ENCODING_MAP(WIDE(kA64Add4rrro), SF_VARIANTS(0x0b000000), 121 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 122 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12, 123 "add", "!0r, !1r, !2r!3o", kFixupNone), 124 ENCODING_MAP(WIDE(kA64Add4RRre), SF_VARIANTS(0x0b200000), 125 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16, 126 kFmtExtend, -1, -1, IS_QUAD_OP | REG_DEF0_USE12, 127 "add", "!0r, !1r, !2r!3e", kFixupNone), 128 // Note: adr is binary, but declared as tertiary. The third argument is used while doing the 129 // fixups and contains information to identify the adr label. 130 ENCODING_MAP(kA64Adr2xd, NO_VARIANTS(0x10000000), 131 kFmtRegX, 4, 0, kFmtImm21, -1, -1, kFmtUnused, -1, -1, 132 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0 | NEEDS_FIXUP, 133 "adr", "!0x, #!1d", kFixupAdr), 134 ENCODING_MAP(kA64Adrp2xd, NO_VARIANTS(0x90000000), 135 kFmtRegX, 4, 0, kFmtImm21, -1, -1, kFmtUnused, -1, -1, 136 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0 | NEEDS_FIXUP, 137 "adrp", "!0x, #!1d", kFixupLabel), 138 ENCODING_MAP(WIDE(kA64And3Rrl), SF_VARIANTS(0x12000000), 139 kFmtRegROrSp, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 22, 10, 140 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1, 141 "and", "!0R, !1r, #!2l", kFixupNone), 142 ENCODING_MAP(WIDE(kA64And4rrro), SF_VARIANTS(0x0a000000), 143 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 144 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12, 145 "and", "!0r, !1r, !2r!3o", kFixupNone), 146 ENCODING_MAP(WIDE(kA64Asr3rrd), CUSTOM_VARIANTS(0x13007c00, 0x9340fc00), 147 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16, 148 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1, 149 "asr", "!0r, !1r, #!2d", kFixupNone), 150 ENCODING_MAP(WIDE(kA64Asr3rrr), SF_VARIANTS(0x1ac02800), 151 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 152 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 153 "asr", "!0r, !1r, !2r", kFixupNone), 154 ENCODING_MAP(kA64B2ct, NO_VARIANTS(0x54000000), 155 kFmtBitBlt, 3, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1, 156 kFmtUnused, -1, -1, IS_BINARY_OP | IS_BRANCH | USES_CCODES | 157 NEEDS_FIXUP, "b.!0c", "!1t", kFixupCondBranch), 158 ENCODING_MAP(kA64Blr1x, NO_VARIANTS(0xd63f0000), 159 kFmtRegX, 9, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 160 kFmtUnused, -1, -1, 161 IS_UNARY_OP | REG_USE0 | IS_BRANCH | REG_DEF_LR, 162 "blr", "!0x", kFixupNone), 163 ENCODING_MAP(kA64Br1x, NO_VARIANTS(0xd61f0000), 164 kFmtRegX, 9, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 165 kFmtUnused, -1, -1, IS_UNARY_OP | REG_USE0 | IS_BRANCH, 166 "br", "!0x", kFixupNone), 167 ENCODING_MAP(kA64Bl1t, NO_VARIANTS(0x94000000), 168 kFmtBitBlt, 25, 0, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 169 kFmtUnused, -1, -1, IS_UNARY_OP | IS_BRANCH | REG_DEF_LR | NEEDS_FIXUP, 170 "bl", "!0T", kFixupLabel), 171 ENCODING_MAP(kA64Brk1d, NO_VARIANTS(0xd4200000), 172 kFmtBitBlt, 20, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 173 kFmtUnused, -1, -1, IS_UNARY_OP | IS_BRANCH, 174 "brk", "!0d", kFixupNone), 175 ENCODING_MAP(kA64B1t, NO_VARIANTS(0x14000000), 176 kFmtBitBlt, 25, 0, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 177 kFmtUnused, -1, -1, IS_UNARY_OP | IS_BRANCH | NEEDS_FIXUP, 178 "b", "!0t", kFixupT1Branch), 179 ENCODING_MAP(WIDE(kA64Cbnz2rt), SF_VARIANTS(0x35000000), 180 kFmtRegR, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1, 181 kFmtUnused, -1, -1, 182 IS_BINARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP, 183 "cbnz", "!0r, !1t", kFixupCBxZ), 184 ENCODING_MAP(WIDE(kA64Cbz2rt), SF_VARIANTS(0x34000000), 185 kFmtRegR, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1, 186 kFmtUnused, -1, -1, 187 IS_BINARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP, 188 "cbz", "!0r, !1t", kFixupCBxZ), 189 ENCODING_MAP(WIDE(kA64Cmn3rro), SF_VARIANTS(0x2b00001f), 190 kFmtRegR, 9, 5, kFmtRegR, 20, 16, kFmtShift, -1, -1, 191 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES, 192 "cmn", "!0r, !1r!2o", kFixupNone), 193 ENCODING_MAP(WIDE(kA64Cmn3Rre), SF_VARIANTS(0x2b20001f), 194 kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16, kFmtExtend, -1, -1, 195 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES, 196 "cmn", "!0R, !1r!2e", kFixupNone), 197 ENCODING_MAP(WIDE(kA64Cmn3RdT), SF_VARIANTS(0x3100001f), 198 kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10, kFmtBitBlt, 23, 22, 199 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE0 | SETS_CCODES, 200 "cmn", "!0R, #!1d!2T", kFixupNone), 201 ENCODING_MAP(WIDE(kA64Cmp3rro), SF_VARIANTS(0x6b00001f), 202 kFmtRegR, 9, 5, kFmtRegR, 20, 16, kFmtShift, -1, -1, 203 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES, 204 "cmp", "!0r, !1r!2o", kFixupNone), 205 ENCODING_MAP(WIDE(kA64Cmp3Rre), SF_VARIANTS(0x6b20001f), 206 kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16, kFmtExtend, -1, -1, 207 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES, 208 "cmp", "!0R, !1r!2e", kFixupNone), 209 ENCODING_MAP(WIDE(kA64Cmp3RdT), SF_VARIANTS(0x7100001f), 210 kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10, kFmtBitBlt, 23, 22, 211 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE0 | SETS_CCODES, 212 "cmp", "!0R, #!1d!2T", kFixupNone), 213 ENCODING_MAP(WIDE(kA64Csel4rrrc), SF_VARIANTS(0x1a800000), 214 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 215 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES, 216 "csel", "!0r, !1r, !2r, !3c", kFixupNone), 217 ENCODING_MAP(WIDE(kA64Csinc4rrrc), SF_VARIANTS(0x1a800400), 218 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 219 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES, 220 "csinc", "!0r, !1r, !2r, !3c", kFixupNone), 221 ENCODING_MAP(WIDE(kA64Csinv4rrrc), SF_VARIANTS(0x5a800000), 222 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 223 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES, 224 "csinv", "!0r, !1r, !2r, !3c", kFixupNone), 225 ENCODING_MAP(WIDE(kA64Csneg4rrrc), SF_VARIANTS(0x5a800400), 226 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 227 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES, 228 "csneg", "!0r, !1r, !2r, !3c", kFixupNone), 229 ENCODING_MAP(kA64Dmb1B, NO_VARIANTS(0xd50330bf), 230 kFmtBitBlt, 11, 8, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 231 kFmtUnused, -1, -1, IS_UNARY_OP | IS_VOLATILE, 232 "dmb", "#!0B", kFixupNone), 233 ENCODING_MAP(WIDE(kA64Eor3Rrl), SF_VARIANTS(0x52000000), 234 kFmtRegROrSp, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 22, 10, 235 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1, 236 "eor", "!0R, !1r, #!2l", kFixupNone), 237 ENCODING_MAP(WIDE(kA64Eor4rrro), SF_VARIANTS(0x4a000000), 238 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 239 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12, 240 "eor", "!0r, !1r, !2r!3o", kFixupNone), 241 ENCODING_MAP(WIDE(kA64Extr4rrrd), SF_N_VARIANTS(0x13800000), 242 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 243 kFmtBitBlt, 15, 10, IS_QUAD_OP | REG_DEF0_USE12, 244 "extr", "!0r, !1r, !2r, #!3d", kFixupNone), 245 ENCODING_MAP(WIDE(kA64Fabs2ff), FLOAT_VARIANTS(0x1e20c000), 246 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 247 kFmtUnused, -1, -1, IS_BINARY_OP| REG_DEF0_USE1, 248 "fabs", "!0f, !1f", kFixupNone), 249 ENCODING_MAP(WIDE(kA64Fadd3fff), FLOAT_VARIANTS(0x1e202800), 250 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16, 251 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 252 "fadd", "!0f, !1f, !2f", kFixupNone), 253 ENCODING_MAP(WIDE(kA64Fcmp1f), FLOAT_VARIANTS(0x1e202008), 254 kFmtRegF, 9, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 255 kFmtUnused, -1, -1, IS_UNARY_OP | REG_USE0 | SETS_CCODES, 256 "fcmp", "!0f, #0", kFixupNone), 257 ENCODING_MAP(WIDE(kA64Fcmp2ff), FLOAT_VARIANTS(0x1e202000), 258 kFmtRegF, 9, 5, kFmtRegF, 20, 16, kFmtUnused, -1, -1, 259 kFmtUnused, -1, -1, IS_BINARY_OP | REG_USE01 | SETS_CCODES, 260 "fcmp", "!0f, !1f", kFixupNone), 261 ENCODING_MAP(WIDE(kA64Fcvtzs2wf), FLOAT_VARIANTS(0x1e380000), 262 kFmtRegW, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 263 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 264 "fcvtzs", "!0w, !1f", kFixupNone), 265 ENCODING_MAP(WIDE(kA64Fcvtzs2xf), FLOAT_VARIANTS(0x9e380000), 266 kFmtRegX, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 267 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 268 "fcvtzs", "!0x, !1f", kFixupNone), 269 ENCODING_MAP(kA64Fcvt2Ss, NO_VARIANTS(0x1e22C000), 270 kFmtRegD, 4, 0, kFmtRegS, 9, 5, kFmtUnused, -1, -1, 271 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 272 "fcvt", "!0S, !1s", kFixupNone), 273 ENCODING_MAP(kA64Fcvt2sS, NO_VARIANTS(0x1e624000), 274 kFmtRegS, 4, 0, kFmtRegD, 9, 5, kFmtUnused, -1, -1, 275 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 276 "fcvt", "!0s, !1S", kFixupNone), 277 ENCODING_MAP(kA64Fcvtms2ws, NO_VARIANTS(0x1e300000), 278 kFmtRegW, 4, 0, kFmtRegS, 9, 5, kFmtUnused, -1, -1, 279 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 280 "fcvtms", "!0w, !1s", kFixupNone), 281 ENCODING_MAP(kA64Fcvtms2xS, NO_VARIANTS(0x9e700000), 282 kFmtRegX, 4, 0, kFmtRegD, 9, 5, kFmtUnused, -1, -1, 283 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 284 "fcvtms", "!0x, !1S", kFixupNone), 285 ENCODING_MAP(WIDE(kA64Fdiv3fff), FLOAT_VARIANTS(0x1e201800), 286 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16, 287 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 288 "fdiv", "!0f, !1f, !2f", kFixupNone), 289 ENCODING_MAP(WIDE(kA64Fmax3fff), FLOAT_VARIANTS(0x1e204800), 290 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16, 291 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 292 "fmax", "!0f, !1f, !2f", kFixupNone), 293 ENCODING_MAP(WIDE(kA64Fmin3fff), FLOAT_VARIANTS(0x1e205800), 294 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16, 295 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 296 "fmin", "!0f, !1f, !2f", kFixupNone), 297 ENCODING_MAP(WIDE(kA64Fmov2ff), FLOAT_VARIANTS(0x1e204000), 298 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 299 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_MOVE, 300 "fmov", "!0f, !1f", kFixupNone), 301 ENCODING_MAP(WIDE(kA64Fmov2fI), FLOAT_VARIANTS(0x1e201000), 302 kFmtRegF, 4, 0, kFmtBitBlt, 20, 13, kFmtUnused, -1, -1, 303 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0, 304 "fmov", "!0f, #!1I", kFixupNone), 305 ENCODING_MAP(kA64Fmov2sw, NO_VARIANTS(0x1e270000), 306 kFmtRegS, 4, 0, kFmtRegW, 9, 5, kFmtUnused, -1, -1, 307 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 308 "fmov", "!0s, !1w", kFixupNone), 309 ENCODING_MAP(kA64Fmov2Sx, NO_VARIANTS(0x9e670000), 310 kFmtRegD, 4, 0, kFmtRegX, 9, 5, kFmtUnused, -1, -1, 311 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 312 "fmov", "!0S, !1x", kFixupNone), 313 ENCODING_MAP(kA64Fmov2ws, NO_VARIANTS(0x1e260000), 314 kFmtRegW, 4, 0, kFmtRegS, 9, 5, kFmtUnused, -1, -1, 315 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 316 "fmov", "!0w, !1s", kFixupNone), 317 ENCODING_MAP(kA64Fmov2xS, NO_VARIANTS(0x9e660000), 318 kFmtRegX, 4, 0, kFmtRegD, 9, 5, kFmtUnused, -1, -1, 319 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 320 "fmov", "!0x, !1S", kFixupNone), 321 ENCODING_MAP(WIDE(kA64Fmul3fff), FLOAT_VARIANTS(0x1e200800), 322 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16, 323 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 324 "fmul", "!0f, !1f, !2f", kFixupNone), 325 ENCODING_MAP(WIDE(kA64Fneg2ff), FLOAT_VARIANTS(0x1e214000), 326 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 327 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 328 "fneg", "!0f, !1f", kFixupNone), 329 ENCODING_MAP(WIDE(kA64Frintp2ff), FLOAT_VARIANTS(0x1e24c000), 330 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 331 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 332 "frintp", "!0f, !1f", kFixupNone), 333 ENCODING_MAP(WIDE(kA64Frintm2ff), FLOAT_VARIANTS(0x1e254000), 334 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 335 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 336 "frintm", "!0f, !1f", kFixupNone), 337 ENCODING_MAP(WIDE(kA64Frintn2ff), FLOAT_VARIANTS(0x1e244000), 338 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 339 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 340 "frintn", "!0f, !1f", kFixupNone), 341 ENCODING_MAP(WIDE(kA64Frintz2ff), FLOAT_VARIANTS(0x1e25c000), 342 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 343 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 344 "frintz", "!0f, !1f", kFixupNone), 345 ENCODING_MAP(WIDE(kA64Fsqrt2ff), FLOAT_VARIANTS(0x1e61c000), 346 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1, 347 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 348 "fsqrt", "!0f, !1f", kFixupNone), 349 ENCODING_MAP(WIDE(kA64Fsub3fff), FLOAT_VARIANTS(0x1e203800), 350 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16, 351 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 352 "fsub", "!0f, !1f, !2f", kFixupNone), 353 ENCODING_MAP(kA64Ldrb3wXd, NO_VARIANTS(0x39400000), 354 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 355 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF, 356 "ldrb", "!0w, [!1X, #!2d]", kFixupNone), 357 ENCODING_MAP(kA64Ldrb3wXx, NO_VARIANTS(0x38606800), 358 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 359 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_LOAD, 360 "ldrb", "!0w, [!1X, !2x]", kFixupNone), 361 ENCODING_MAP(WIDE(kA64Ldrsb3rXd), CUSTOM_VARIANTS(0x39c00000, 0x39800000), 362 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 363 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF, 364 "ldrsb", "!0r, [!1X, #!2d]", kFixupNone), 365 ENCODING_MAP(WIDE(kA64Ldrsb3rXx), CUSTOM_VARIANTS(0x38e06800, 0x38a06800), 366 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 367 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_LOAD, 368 "ldrsb", "!0r, [!1X, !2x]", kFixupNone), 369 ENCODING_MAP(kA64Ldrh3wXF, NO_VARIANTS(0x79400000), 370 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 371 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF, 372 "ldrh", "!0w, [!1X, #!2F]", kFixupNone), 373 ENCODING_MAP(kA64Ldrh4wXxd, NO_VARIANTS(0x78606800), 374 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 375 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD_OFF, 376 "ldrh", "!0w, [!1X, !2x, lsl #!3d]", kFixupNone), 377 ENCODING_MAP(WIDE(kA64Ldrsh3rXF), CUSTOM_VARIANTS(0x79c00000, 0x79800000), 378 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 379 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF, 380 "ldrsh", "!0r, [!1X, #!2F]", kFixupNone), 381 ENCODING_MAP(WIDE(kA64Ldrsh4rXxd), CUSTOM_VARIANTS(0x78e06800, 0x78906800), 382 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 383 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD_OFF, 384 "ldrsh", "!0r, [!1X, !2x, lsl #!3d]", kFixupNone), 385 ENCODING_MAP(WIDE(kA64Ldr2fp), SIZE_VARIANTS(0x1c000000), 386 kFmtRegF, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1, 387 kFmtUnused, -1, -1, 388 IS_BINARY_OP | REG_DEF0 | REG_USE_PC | IS_LOAD | NEEDS_FIXUP, 389 "ldr", "!0f, !1p", kFixupLoad), 390 ENCODING_MAP(WIDE(kA64Ldr2rp), SIZE_VARIANTS(0x18000000), 391 kFmtRegR, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1, 392 kFmtUnused, -1, -1, 393 IS_BINARY_OP | REG_DEF0 | REG_USE_PC | IS_LOAD | NEEDS_FIXUP, 394 "ldr", "!0r, !1p", kFixupLoad), 395 ENCODING_MAP(WIDE(kA64Ldr3fXD), SIZE_VARIANTS(0xbd400000), 396 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 397 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF, 398 "ldr", "!0f, [!1X, #!2D]", kFixupNone), 399 ENCODING_MAP(WIDE(kA64Ldr3rXD), SIZE_VARIANTS(0xb9400000), 400 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 401 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF, 402 "ldr", "!0r, [!1X, #!2D]", kFixupNone), 403 ENCODING_MAP(WIDE(kA64Ldr4fXxG), SIZE_VARIANTS(0xbc606800), 404 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 405 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD, 406 "ldr", "!0f, [!1X, !2x!3G]", kFixupNone), 407 ENCODING_MAP(WIDE(kA64Ldr4rXxG), SIZE_VARIANTS(0xb8606800), 408 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 409 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD, 410 "ldr", "!0r, [!1X, !2x!3G]", kFixupNone), 411 ENCODING_MAP(WIDE(kA64LdrPost3rXd), SIZE_VARIANTS(0xb8400400), 412 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12, 413 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF01 | REG_USE1 | IS_LOAD, 414 "ldr", "!0r, [!1X], #!2d", kFixupNone), 415 ENCODING_MAP(WIDE(kA64Ldp4ffXD), CUSTOM_VARIANTS(0x2d400000, 0x6d400000), 416 kFmtRegF, 4, 0, kFmtRegF, 14, 10, kFmtRegXOrSp, 9, 5, 417 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE2 | REG_DEF01 | IS_LOAD_OFF, 418 "ldp", "!0f, !1f, [!2X, #!3D]", kFixupNone), 419 ENCODING_MAP(WIDE(kA64Ldp4rrXD), SF_VARIANTS(0x29400000), 420 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5, 421 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE2 | REG_DEF01 | IS_LOAD_OFF, 422 "ldp", "!0r, !1r, [!2X, #!3D]", kFixupNone), 423 ENCODING_MAP(WIDE(kA64LdpPost4rrXD), CUSTOM_VARIANTS(0x28c00000, 0xa8c00000), 424 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5, 425 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE2 | REG_DEF012 | IS_LOAD, 426 "ldp", "!0r, !1r, [!2X], #!3D", kFixupNone), 427 ENCODING_MAP(WIDE(kA64Ldur3fXd), CUSTOM_VARIANTS(0xbc400000, 0xfc400000), 428 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12, 429 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD, 430 "ldur", "!0f, [!1X, #!2d]", kFixupNone), 431 ENCODING_MAP(WIDE(kA64Ldur3rXd), SIZE_VARIANTS(0xb8400000), 432 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12, 433 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD, 434 "ldur", "!0r, [!1X, #!2d]", kFixupNone), 435 ENCODING_MAP(WIDE(kA64Ldxr2rX), SIZE_VARIANTS(0x885f7c00), 436 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtUnused, -1, -1, 437 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_LOADX, 438 "ldxr", "!0r, [!1X]", kFixupNone), 439 ENCODING_MAP(WIDE(kA64Ldaxr2rX), SIZE_VARIANTS(0x885ffc00), 440 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtUnused, -1, -1, 441 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_LOADX, 442 "ldaxr", "!0r, [!1X]", kFixupNone), 443 ENCODING_MAP(WIDE(kA64Lsl3rrr), SF_VARIANTS(0x1ac02000), 444 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 445 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 446 "lsl", "!0r, !1r, !2r", kFixupNone), 447 ENCODING_MAP(WIDE(kA64Lsr3rrd), CUSTOM_VARIANTS(0x53007c00, 0xd340fc00), 448 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16, 449 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1, 450 "lsr", "!0r, !1r, #!2d", kFixupNone), 451 ENCODING_MAP(WIDE(kA64Lsr3rrr), SF_VARIANTS(0x1ac02400), 452 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 453 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 454 "lsr", "!0r, !1r, !2r", kFixupNone), 455 ENCODING_MAP(WIDE(kA64Madd4rrrr), SF_VARIANTS(0x1b000000), 456 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 457 kFmtRegR, 14, 10, IS_QUAD_OP | REG_DEF0_USE123 | NEEDS_FIXUP, 458 "madd", "!0r, !1r, !2r, !3r", kFixupA53Erratum835769), 459 ENCODING_MAP(WIDE(kA64Movk3rdM), SF_VARIANTS(0x72800000), 460 kFmtRegR, 4, 0, kFmtBitBlt, 20, 5, kFmtBitBlt, 22, 21, 461 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE0, 462 "movk", "!0r, #!1d!2M", kFixupNone), 463 ENCODING_MAP(WIDE(kA64Movn3rdM), SF_VARIANTS(0x12800000), 464 kFmtRegR, 4, 0, kFmtBitBlt, 20, 5, kFmtBitBlt, 22, 21, 465 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0, 466 "movn", "!0r, #!1d!2M", kFixupNone), 467 ENCODING_MAP(WIDE(kA64Movz3rdM), SF_VARIANTS(0x52800000), 468 kFmtRegR, 4, 0, kFmtBitBlt, 20, 5, kFmtBitBlt, 22, 21, 469 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0, 470 "movz", "!0r, #!1d!2M", kFixupNone), 471 ENCODING_MAP(WIDE(kA64Mov2rr), SF_VARIANTS(0x2a0003e0), 472 kFmtRegR, 4, 0, kFmtRegR, 20, 16, kFmtUnused, -1, -1, 473 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_MOVE, 474 "mov", "!0r, !1r", kFixupNone), 475 ENCODING_MAP(WIDE(kA64Mvn2rr), SF_VARIANTS(0x2a2003e0), 476 kFmtRegR, 4, 0, kFmtRegR, 20, 16, kFmtUnused, -1, -1, 477 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 478 "mvn", "!0r, !1r", kFixupNone), 479 ENCODING_MAP(WIDE(kA64Mul3rrr), SF_VARIANTS(0x1b007c00), 480 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 481 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 482 "mul", "!0r, !1r, !2r", kFixupNone), 483 ENCODING_MAP(WIDE(kA64Msub4rrrr), SF_VARIANTS(0x1b008000), 484 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 485 kFmtRegR, 14, 10, IS_QUAD_OP | REG_DEF0_USE123 | NEEDS_FIXUP, 486 "msub", "!0r, !1r, !2r, !3r", kFixupA53Erratum835769), 487 ENCODING_MAP(WIDE(kA64Neg3rro), SF_VARIANTS(0x4b0003e0), 488 kFmtRegR, 4, 0, kFmtRegR, 20, 16, kFmtShift, -1, -1, 489 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1, 490 "neg", "!0r, !1r!2o", kFixupNone), 491 ENCODING_MAP(kA64Nop0, NO_VARIANTS(0xd503201f), 492 kFmtUnused, -1, -1, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 493 kFmtUnused, -1, -1, NO_OPERAND, 494 "nop", "", kFixupNone), 495 ENCODING_MAP(WIDE(kA64Orr3Rrl), SF_VARIANTS(0x32000000), 496 kFmtRegROrSp, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 22, 10, 497 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1, 498 "orr", "!0R, !1r, #!2l", kFixupNone), 499 ENCODING_MAP(WIDE(kA64Orr4rrro), SF_VARIANTS(0x2a000000), 500 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 501 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12, 502 "orr", "!0r, !1r, !2r!3o", kFixupNone), 503 ENCODING_MAP(kA64Ret, NO_VARIANTS(0xd65f03c0), 504 kFmtUnused, -1, -1, kFmtUnused, -1, -1, kFmtUnused, -1, -1, 505 kFmtUnused, -1, -1, NO_OPERAND | IS_BRANCH, 506 "ret", "", kFixupNone), 507 ENCODING_MAP(WIDE(kA64Rbit2rr), SF_VARIANTS(0x5ac00000), 508 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtUnused, -1, -1, 509 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 510 "rbit", "!0r, !1r", kFixupNone), 511 ENCODING_MAP(WIDE(kA64Rev2rr), CUSTOM_VARIANTS(0x5ac00800, 0xdac00c00), 512 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtUnused, -1, -1, 513 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 514 "rev", "!0r, !1r", kFixupNone), 515 ENCODING_MAP(WIDE(kA64Rev162rr), SF_VARIANTS(0x5ac00400), 516 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtUnused, -1, -1, 517 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 518 "rev16", "!0r, !1r", kFixupNone), 519 ENCODING_MAP(WIDE(kA64Ror3rrr), SF_VARIANTS(0x1ac02c00), 520 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 521 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 522 "ror", "!0r, !1r, !2r", kFixupNone), 523 ENCODING_MAP(WIDE(kA64Sbc3rrr), SF_VARIANTS(0x5a000000), 524 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 525 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | USES_CCODES, 526 "sbc", "!0r, !1r, !2r", kFixupNone), 527 ENCODING_MAP(WIDE(kA64Sbfm4rrdd), SF_N_VARIANTS(0x13000000), 528 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16, 529 kFmtBitBlt, 15, 10, IS_QUAD_OP | REG_DEF0_USE1, 530 "sbfm", "!0r, !1r, #!2d, #!3d", kFixupNone), 531 ENCODING_MAP(WIDE(kA64Scvtf2fw), FLOAT_VARIANTS(0x1e220000), 532 kFmtRegF, 4, 0, kFmtRegW, 9, 5, kFmtUnused, -1, -1, 533 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 534 "scvtf", "!0f, !1w", kFixupNone), 535 ENCODING_MAP(WIDE(kA64Scvtf2fx), FLOAT_VARIANTS(0x9e220000), 536 kFmtRegF, 4, 0, kFmtRegX, 9, 5, kFmtUnused, -1, -1, 537 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1, 538 "scvtf", "!0f, !1x", kFixupNone), 539 ENCODING_MAP(WIDE(kA64Sdiv3rrr), SF_VARIANTS(0x1ac00c00), 540 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 541 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 542 "sdiv", "!0r, !1r, !2r", kFixupNone), 543 ENCODING_MAP(kA64Smull3xww, NO_VARIANTS(0x9b207c00), 544 kFmtRegX, 4, 0, kFmtRegW, 9, 5, kFmtRegW, 20, 16, 545 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 546 "smull", "!0x, !1w, !2w", kFixupNone), 547 ENCODING_MAP(kA64Smulh3xxx, NO_VARIANTS(0x9b407c00), 548 kFmtRegX, 4, 0, kFmtRegX, 9, 5, kFmtRegX, 20, 16, 549 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12, 550 "smulh", "!0x, !1x, !2x", kFixupNone), 551 ENCODING_MAP(WIDE(kA64Stp4ffXD), CUSTOM_VARIANTS(0x2d000000, 0x6d000000), 552 kFmtRegF, 4, 0, kFmtRegF, 14, 10, kFmtRegXOrSp, 9, 5, 553 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE012 | IS_STORE_OFF, 554 "stp", "!0f, !1f, [!2X, #!3D]", kFixupNone), 555 ENCODING_MAP(WIDE(kA64Stp4rrXD), SF_VARIANTS(0x29000000), 556 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5, 557 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE012 | IS_STORE_OFF, 558 "stp", "!0r, !1r, [!2X, #!3D]", kFixupNone), 559 ENCODING_MAP(WIDE(kA64StpPost4rrXD), CUSTOM_VARIANTS(0x28800000, 0xa8800000), 560 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5, 561 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_DEF2 | REG_USE012 | IS_STORE, 562 "stp", "!0r, !1r, [!2X], #!3D", kFixupNone), 563 ENCODING_MAP(WIDE(kA64StpPre4ffXD), CUSTOM_VARIANTS(0x2d800000, 0x6d800000), 564 kFmtRegF, 4, 0, kFmtRegF, 14, 10, kFmtRegXOrSp, 9, 5, 565 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_DEF2 | REG_USE012 | IS_STORE, 566 "stp", "!0f, !1f, [!2X, #!3D]!!", kFixupNone), 567 ENCODING_MAP(WIDE(kA64StpPre4rrXD), CUSTOM_VARIANTS(0x29800000, 0xa9800000), 568 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5, 569 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_DEF2 | REG_USE012 | IS_STORE, 570 "stp", "!0r, !1r, [!2X, #!3D]!!", kFixupNone), 571 ENCODING_MAP(WIDE(kA64Str3fXD), CUSTOM_VARIANTS(0xbd000000, 0xfd000000), 572 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 573 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF, 574 "str", "!0f, [!1X, #!2D]", kFixupNone), 575 ENCODING_MAP(WIDE(kA64Str4fXxG), CUSTOM_VARIANTS(0xbc206800, 0xfc206800), 576 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 577 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_USE012 | IS_STORE, 578 "str", "!0f, [!1X, !2x!3G]", kFixupNone), 579 ENCODING_MAP(WIDE(kA64Str3rXD), SIZE_VARIANTS(0xb9000000), 580 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 581 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF, 582 "str", "!0r, [!1X, #!2D]", kFixupNone), 583 ENCODING_MAP(WIDE(kA64Str4rXxG), SIZE_VARIANTS(0xb8206800), 584 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 585 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_USE012 | IS_STORE, 586 "str", "!0r, [!1X, !2x!3G]", kFixupNone), 587 ENCODING_MAP(kA64Strb3wXd, NO_VARIANTS(0x39000000), 588 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 589 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF, 590 "strb", "!0w, [!1X, #!2d]", kFixupNone), 591 ENCODING_MAP(kA64Strb3wXx, NO_VARIANTS(0x38206800), 592 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 593 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE012 | IS_STORE, 594 "strb", "!0w, [!1X, !2x]", kFixupNone), 595 ENCODING_MAP(kA64Strh3wXF, NO_VARIANTS(0x79000000), 596 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10, 597 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF, 598 "strh", "!0w, [!1X, #!2F]", kFixupNone), 599 ENCODING_MAP(kA64Strh4wXxd, NO_VARIANTS(0x78206800), 600 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16, 601 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_USE012 | IS_STORE, 602 "strh", "!0w, [!1X, !2x, lsl #!3d]", kFixupNone), 603 ENCODING_MAP(WIDE(kA64StrPost3rXd), SIZE_VARIANTS(0xb8000400), 604 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12, 605 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | REG_DEF1 | IS_STORE, 606 "str", "!0r, [!1X], #!2d", kFixupNone), 607 ENCODING_MAP(WIDE(kA64Stur3fXd), CUSTOM_VARIANTS(0xbc000000, 0xfc000000), 608 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12, 609 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE, 610 "stur", "!0f, [!1X, #!2d]", kFixupNone), 611 ENCODING_MAP(WIDE(kA64Stur3rXd), SIZE_VARIANTS(0xb8000000), 612 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12, 613 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE, 614 "stur", "!0r, [!1X, #!2d]", kFixupNone), 615 ENCODING_MAP(WIDE(kA64Stxr3wrX), SIZE_VARIANTS(0x88007c00), 616 kFmtRegW, 20, 16, kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, 617 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_STOREX, 618 "stxr", "!0w, !1r, [!2X]", kFixupNone), 619 ENCODING_MAP(WIDE(kA64Stlxr3wrX), SIZE_VARIANTS(0x8800fc00), 620 kFmtRegW, 20, 16, kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, 621 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_STOREX, 622 "stlxr", "!0w, !1r, [!2X]", kFixupNone), 623 ENCODING_MAP(WIDE(kA64Sub4RRdT), SF_VARIANTS(0x51000000), 624 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10, 625 kFmtBitBlt, 23, 22, IS_QUAD_OP | REG_DEF0_USE1, 626 "sub", "!0R, !1R, #!2d!3T", kFixupNone), 627 ENCODING_MAP(WIDE(kA64Sub4rrro), SF_VARIANTS(0x4b000000), 628 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16, 629 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12, 630 "sub", "!0r, !1r, !2r!3o", kFixupNone), 631 ENCODING_MAP(WIDE(kA64Sub4RRre), SF_VARIANTS(0x4b200000), 632 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16, 633 kFmtExtend, -1, -1, IS_QUAD_OP | REG_DEF0_USE12, 634 "sub", "!0r, !1r, !2r!3e", kFixupNone), 635 ENCODING_MAP(WIDE(kA64Subs3rRd), SF_VARIANTS(0x71000000), 636 kFmtRegR, 4, 0, kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10, 637 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | SETS_CCODES, 638 "subs", "!0r, !1R, #!2d", kFixupNone), 639 ENCODING_MAP(WIDE(kA64Tst2rl), SF_VARIANTS(0x7200001f), 640 kFmtRegR, 9, 5, kFmtBitBlt, 22, 10, kFmtUnused, -1, -1, 641 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE0 | SETS_CCODES, 642 "tst", "!0r, !1l", kFixupNone), 643 ENCODING_MAP(WIDE(kA64Tst3rro), SF_VARIANTS(0x6a00001f), 644 kFmtRegR, 9, 5, kFmtRegR, 20, 16, kFmtShift, -1, -1, 645 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES, 646 "tst", "!0r, !1r!2o", kFixupNone), 647 // NOTE: Tbz/Tbnz does not require SETS_CCODES, but it may be replaced by some other LIRs 648 // which require SETS_CCODES in the fix-up stage. 649 ENCODING_MAP(WIDE(kA64Tbnz3rht), CUSTOM_VARIANTS(0x37000000, 0x37000000), 650 kFmtRegR, 4, 0, kFmtImm6Shift, -1, -1, kFmtBitBlt, 18, 5, kFmtUnused, -1, -1, 651 IS_TERTIARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP | SETS_CCODES, 652 "tbnz", "!0r, #!1h, !2t", kFixupTBxZ), 653 ENCODING_MAP(WIDE(kA64Tbz3rht), CUSTOM_VARIANTS(0x36000000, 0x36000000), 654 kFmtRegR, 4, 0, kFmtImm6Shift, -1, -1, kFmtBitBlt, 18, 5, kFmtUnused, -1, -1, 655 IS_TERTIARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP | SETS_CCODES, 656 "tbz", "!0r, #!1h, !2t", kFixupTBxZ), 657 ENCODING_MAP(WIDE(kA64Ubfm4rrdd), SF_N_VARIANTS(0x53000000), 658 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16, 659 kFmtBitBlt, 15, 10, IS_QUAD_OP | REG_DEF0_USE1, 660 "ubfm", "!0r, !1r, !2d, !3d", kFixupNone), 661 }; 662 663 // new_lir replaces orig_lir in the pcrel_fixup list. 664 void Arm64Mir2Lir::ReplaceFixup(LIR* prev_lir, LIR* orig_lir, LIR* new_lir) { 665 new_lir->u.a.pcrel_next = orig_lir->u.a.pcrel_next; 666 if (UNLIKELY(prev_lir == nullptr)) { 667 first_fixup_ = new_lir; 668 } else { 669 prev_lir->u.a.pcrel_next = new_lir; 670 } 671 orig_lir->flags.fixup = kFixupNone; 672 } 673 674 // new_lir is inserted before orig_lir in the pcrel_fixup list. 675 void Arm64Mir2Lir::InsertFixupBefore(LIR* prev_lir, LIR* orig_lir, LIR* new_lir) { 676 new_lir->u.a.pcrel_next = orig_lir; 677 if (UNLIKELY(prev_lir == nullptr)) { 678 first_fixup_ = new_lir; 679 } else { 680 DCHECK(prev_lir->u.a.pcrel_next == orig_lir); 681 prev_lir->u.a.pcrel_next = new_lir; 682 } 683 } 684 685 /* Nop, used for aligning code. Nop is an alias for hint #0. */ 686 #define PADDING_NOP (UINT32_C(0xd503201f)) 687 688 uint8_t* Arm64Mir2Lir::EncodeLIRs(uint8_t* write_pos, LIR* lir) { 689 uint8_t* const write_buffer = write_pos; 690 for (; lir != nullptr; lir = NEXT_LIR(lir)) { 691 lir->offset = (write_pos - write_buffer); 692 bool opcode_is_wide = IS_WIDE(lir->opcode); 693 A64Opcode opcode = UNWIDE(lir->opcode); 694 695 if (UNLIKELY(IsPseudoLirOp(opcode))) { 696 continue; 697 } 698 699 if (LIKELY(!lir->flags.is_nop)) { 700 const A64EncodingMap *encoder = &EncodingMap[opcode]; 701 702 // Select the right variant of the skeleton. 703 uint32_t bits = opcode_is_wide ? encoder->xskeleton : encoder->wskeleton; 704 DCHECK(!opcode_is_wide || IS_WIDE(encoder->opcode)); 705 706 for (int i = 0; i < 4; i++) { 707 A64EncodingKind kind = encoder->field_loc[i].kind; 708 uint32_t operand = lir->operands[i]; 709 uint32_t value; 710 711 if (LIKELY(static_cast<unsigned>(kind) <= kFmtBitBlt)) { 712 // Note: this will handle kFmtReg* and kFmtBitBlt. 713 714 if (static_cast<unsigned>(kind) < kFmtBitBlt) { 715 bool is_zero = A64_REG_IS_ZR(operand); 716 717 if (kIsDebugBuild && (kFailOnSizeError || kReportSizeError)) { 718 // Register usage checks: First establish register usage requirements based on the 719 // format in `kind'. 720 bool want_float = false; // Want a float (rather than core) register. 721 bool want_64_bit = false; // Want a 64-bit (rather than 32-bit) register. 722 bool want_var_size = true; // Want register with variable size (kFmtReg{R,F}). 723 bool want_zero = false; // Want the zero (rather than sp) register. 724 switch (kind) { 725 case kFmtRegX: 726 want_64_bit = true; 727 FALLTHROUGH_INTENDED; 728 case kFmtRegW: 729 want_var_size = false; 730 FALLTHROUGH_INTENDED; 731 case kFmtRegR: 732 want_zero = true; 733 break; 734 case kFmtRegXOrSp: 735 want_64_bit = true; 736 FALLTHROUGH_INTENDED; 737 case kFmtRegWOrSp: 738 want_var_size = false; 739 break; 740 case kFmtRegROrSp: 741 break; 742 case kFmtRegD: 743 want_64_bit = true; 744 FALLTHROUGH_INTENDED; 745 case kFmtRegS: 746 want_var_size = false; 747 FALLTHROUGH_INTENDED; 748 case kFmtRegF: 749 want_float = true; 750 break; 751 default: 752 LOG(FATAL) << "Bad fmt for arg n. " << i << " of " << encoder->name 753 << " (" << kind << ")"; 754 break; 755 } 756 757 // want_var_size == true means kind == kFmtReg{R,F}. In these two cases, we want 758 // the register size to be coherent with the instruction width. 759 if (want_var_size) { 760 want_64_bit = opcode_is_wide; 761 } 762 763 // Now check that the requirements are satisfied. 764 RegStorage reg(operand | RegStorage::kValid); 765 const char *expected = nullptr; 766 if (want_float) { 767 if (!reg.IsFloat()) { 768 expected = "float register"; 769 } else if (reg.IsDouble() != want_64_bit) { 770 expected = (want_64_bit) ? "double register" : "single register"; 771 } 772 } else { 773 if (reg.IsFloat()) { 774 expected = "core register"; 775 } else if (reg.Is64Bit() != want_64_bit) { 776 expected = (want_64_bit) ? "x-register" : "w-register"; 777 } else if (A64_REGSTORAGE_IS_SP_OR_ZR(reg) && is_zero != want_zero) { 778 expected = (want_zero) ? "zero-register" : "sp-register"; 779 } 780 } 781 782 // Fail, if `expected' contains an unsatisfied requirement. 783 if (expected != nullptr) { 784 LOG(WARNING) << "Method: " << PrettyMethod(cu_->method_idx, *cu_->dex_file) 785 << " @ 0x" << std::hex << lir->dalvik_offset; 786 if (kFailOnSizeError) { 787 LOG(FATAL) << "Bad argument n. " << i << " of " << encoder->name 788 << "(" << UNWIDE(encoder->opcode) << ", " << encoder->fmt << ")" 789 << ". Expected " << expected << ", got 0x" << std::hex << operand; 790 } else { 791 LOG(WARNING) << "Bad argument n. " << i << " of " << encoder->name 792 << ". Expected " << expected << ", got 0x" << std::hex << operand; 793 } 794 } 795 } 796 797 // In the lines below, we rely on (operand & 0x1f) == 31 to be true for register sp 798 // and zr. This means that these two registers do not need any special treatment, as 799 // their bottom 5 bits are correctly set to 31 == 0b11111, which is the right 800 // value for encoding both sp and zr. 801 static_assert((rxzr & 0x1f) == 0x1f, "rzr register number must be 31"); 802 static_assert((rsp & 0x1f) == 0x1f, "rsp register number must be 31"); 803 } 804 805 value = (operand << encoder->field_loc[i].start) & 806 ((1 << (encoder->field_loc[i].end + 1)) - 1); 807 bits |= value; 808 } else { 809 switch (kind) { 810 case kFmtSkip: 811 break; // Nothing to do, but continue to next. 812 case kFmtUnused: 813 i = 4; // Done, break out of the enclosing loop. 814 break; 815 case kFmtShift: 816 // Intentional fallthrough. 817 case kFmtExtend: 818 DCHECK_EQ((operand & (1 << 6)) == 0, kind == kFmtShift); 819 value = (operand & 0x3f) << 10; 820 value |= ((operand & 0x1c0) >> 6) << 21; 821 bits |= value; 822 break; 823 case kFmtImm21: 824 value = (operand & 0x3) << 29; 825 value |= ((operand & 0x1ffffc) >> 2) << 5; 826 bits |= value; 827 break; 828 case kFmtImm6Shift: 829 value = (operand & 0x1f) << 19; 830 value |= ((operand & 0x20) >> 5) << 31; 831 bits |= value; 832 break; 833 default: 834 LOG(FATAL) << "Bad fmt for arg. " << i << " in " << encoder->name 835 << " (" << kind << ")"; 836 } 837 } 838 } 839 840 DCHECK_EQ(encoder->size, 4); 841 write_pos[0] = (bits & 0xff); 842 write_pos[1] = ((bits >> 8) & 0xff); 843 write_pos[2] = ((bits >> 16) & 0xff); 844 write_pos[3] = ((bits >> 24) & 0xff); 845 write_pos += 4; 846 } 847 } 848 849 return write_pos; 850 } 851 852 // Align data offset on 8 byte boundary: it will only contain double-word items, as word immediates 853 // are better set directly from the code (they will require no more than 2 instructions). 854 #define ALIGNED_DATA_OFFSET(offset) (((offset) + 0x7) & ~0x7) 855 856 /* 857 * Get the LIR which emits the instruction preceding the given LIR. 858 * Returns nullptr, if no previous emitting insn found. 859 */ 860 static LIR* GetPrevEmittingLIR(LIR* lir) { 861 DCHECK(lir != nullptr); 862 LIR* prev_lir = lir->prev; 863 while ((prev_lir != nullptr) && 864 (prev_lir->flags.is_nop || Mir2Lir::IsPseudoLirOp(prev_lir->opcode))) { 865 prev_lir = prev_lir->prev; 866 } 867 return prev_lir; 868 } 869 870 // Assemble the LIR into binary instruction format. 871 void Arm64Mir2Lir::AssembleLIR() { 872 LIR* lir; 873 LIR* prev_lir; 874 cu_->NewTimingSplit("Assemble"); 875 int assembler_retries = 0; 876 CodeOffset starting_offset = LinkFixupInsns(first_lir_insn_, last_lir_insn_, 0); 877 data_offset_ = ALIGNED_DATA_OFFSET(starting_offset); 878 int32_t offset_adjustment; 879 AssignDataOffsets(); 880 881 /* 882 * Note: generation must be 1 on first pass (to distinguish from initialized state of 0 883 * for non-visited nodes). Start at zero here, and bit will be flipped to 1 on entry to the loop. 884 */ 885 int generation = 0; 886 while (true) { 887 offset_adjustment = 0; 888 AssemblerStatus res = kSuccess; // Assume success 889 generation ^= 1; 890 // Note: nodes requiring possible fixup linked in ascending order. 891 lir = first_fixup_; 892 prev_lir = nullptr; 893 while (lir != nullptr) { 894 // NOTE: Any new non-pc_rel instructions inserted due to retry must be explicitly encoded at 895 // the time of insertion. Note that inserted instructions don't need use/def flags, but do 896 // need size and pc-rel status properly updated. 897 lir->offset += offset_adjustment; 898 // During pass, allows us to tell whether a node has been updated with offset_adjustment yet. 899 lir->flags.generation = generation; 900 switch (static_cast<FixupKind>(lir->flags.fixup)) { 901 case kFixupLabel: 902 case kFixupNone: 903 case kFixupVLoad: 904 break; 905 case kFixupT1Branch: { 906 LIR *target_lir = lir->target; 907 DCHECK(target_lir); 908 CodeOffset pc = lir->offset; 909 CodeOffset target = target_lir->offset + 910 ((target_lir->flags.generation == lir->flags.generation) ? 0 : offset_adjustment); 911 int32_t delta = target - pc; 912 DCHECK_EQ(delta & 0x3, 0); 913 if (!IS_SIGNED_IMM26(delta >> 2)) { 914 LOG(FATAL) << "Invalid jump range in kFixupT1Branch"; 915 } 916 lir->operands[0] = delta >> 2; 917 if (!(cu_->disable_opt & (1 << kSafeOptimizations)) && lir->operands[0] == 1) { 918 // Useless branch. 919 offset_adjustment -= lir->flags.size; 920 lir->flags.is_nop = true; 921 // Don't unlink - just set to do-nothing. 922 lir->flags.fixup = kFixupNone; 923 res = kRetryAll; 924 } 925 break; 926 } 927 case kFixupLoad: 928 case kFixupCBxZ: 929 case kFixupCondBranch: { 930 LIR *target_lir = lir->target; 931 DCHECK(target_lir); 932 CodeOffset pc = lir->offset; 933 CodeOffset target = target_lir->offset + 934 ((target_lir->flags.generation == lir->flags.generation) ? 0 : offset_adjustment); 935 int32_t delta = target - pc; 936 DCHECK_EQ(delta & 0x3, 0); 937 if (!IS_SIGNED_IMM19(delta >> 2)) { 938 LOG(FATAL) << "Invalid jump range in kFixupLoad"; 939 } 940 lir->operands[1] = delta >> 2; 941 break; 942 } 943 case kFixupTBxZ: { 944 int16_t opcode = lir->opcode; 945 RegStorage reg(lir->operands[0] | RegStorage::kValid); 946 int32_t imm = lir->operands[1]; 947 DCHECK_EQ(IS_WIDE(opcode), reg.Is64Bit()); 948 DCHECK_LT(imm, 64); 949 if (imm >= 32) { 950 DCHECK(IS_WIDE(opcode)); 951 } else if (kIsDebugBuild && IS_WIDE(opcode)) { 952 // "tbz/tbnz x0, #imm(<32)" is the same with "tbz/tbnz w0, #imm(<32)", but GCC/oatdump 953 // will disassemble it as "tbz/tbnz w0, #imm(<32)". So unwide the LIR to make the 954 // compiler log behave the same with those disassembler in debug build. 955 // This will also affect tst instruction if it need to be replaced, but there is no 956 // performance difference between "tst Xt" and "tst Wt". 957 lir->opcode = UNWIDE(opcode); 958 lir->operands[0] = As32BitReg(reg).GetReg(); 959 } 960 961 // Fix-up branch offset. 962 LIR *target_lir = lir->target; 963 DCHECK(target_lir); 964 CodeOffset pc = lir->offset; 965 CodeOffset target = target_lir->offset + 966 ((target_lir->flags.generation == lir->flags.generation) ? 0 : offset_adjustment); 967 int32_t delta = target - pc; 968 DCHECK_EQ(delta & 0x3, 0); 969 // Check if branch offset can be encoded in tbz/tbnz. 970 if (!IS_SIGNED_IMM14(delta >> 2)) { 971 DexOffset dalvik_offset = lir->dalvik_offset; 972 LIR* targetLIR = lir->target; 973 // "tbz/tbnz Rt, #imm, label" -> "tst Rt, #(1<<imm)". 974 offset_adjustment -= lir->flags.size; 975 int32_t encodedImm = EncodeLogicalImmediate(IS_WIDE(opcode), 1 << lir->operands[1]); 976 DCHECK_NE(encodedImm, -1); 977 lir->opcode = IS_WIDE(opcode) ? WIDE(kA64Tst2rl) : kA64Tst2rl; 978 lir->operands[1] = encodedImm; 979 lir->target = nullptr; 980 lir->flags.fixup = EncodingMap[kA64Tst2rl].fixup; 981 lir->flags.size = EncodingMap[kA64Tst2rl].size; 982 offset_adjustment += lir->flags.size; 983 // Insert "beq/bneq label". 984 opcode = UNWIDE(opcode); 985 DCHECK(opcode == kA64Tbz3rht || opcode == kA64Tbnz3rht); 986 LIR* new_lir = RawLIR(dalvik_offset, kA64B2ct, 987 opcode == kA64Tbz3rht ? kArmCondEq : kArmCondNe, 0, 0, 0, 0, targetLIR); 988 InsertLIRAfter(lir, new_lir); 989 new_lir->offset = lir->offset + lir->flags.size; 990 new_lir->flags.generation = generation; 991 new_lir->flags.fixup = EncodingMap[kA64B2ct].fixup; 992 new_lir->flags.size = EncodingMap[kA64B2ct].size; 993 offset_adjustment += new_lir->flags.size; 994 // lir no longer pcrel, unlink and link in new_lir. 995 ReplaceFixup(prev_lir, lir, new_lir); 996 prev_lir = new_lir; // Continue with the new instruction. 997 lir = new_lir->u.a.pcrel_next; 998 res = kRetryAll; 999 continue; 1000 } 1001 lir->operands[2] = delta >> 2; 1002 break; 1003 } 1004 case kFixupAdr: { 1005 LIR* target_lir = lir->target; 1006 int32_t delta; 1007 if (target_lir) { 1008 CodeOffset target_offs = ((target_lir->flags.generation == lir->flags.generation) ? 1009 0 : offset_adjustment) + target_lir->offset; 1010 delta = target_offs - lir->offset; 1011 } else if (lir->operands[2] >= 0) { 1012 const EmbeddedData* tab = UnwrapPointer<EmbeddedData>(lir->operands[2]); 1013 delta = tab->offset + offset_adjustment - lir->offset; 1014 } else { 1015 // No fixup: this usage allows to retrieve the current PC. 1016 delta = lir->operands[1]; 1017 } 1018 if (!IS_SIGNED_IMM21(delta)) { 1019 LOG(FATAL) << "Jump range above 1MB in kFixupAdr"; 1020 } 1021 lir->operands[1] = delta; 1022 break; 1023 } 1024 case kFixupA53Erratum835769: 1025 // Avoid emitting code that could trigger Cortex A53's erratum 835769. 1026 // This fixup should be carried out for all multiply-accumulate instructions: madd, msub, 1027 // smaddl, smsubl, umaddl and umsubl. 1028 if (cu_->compiler_driver->GetInstructionSetFeatures()->AsArm64InstructionSetFeatures() 1029 ->NeedFixCortexA53_835769()) { 1030 // Check that this is a 64-bit multiply-accumulate. 1031 if (IS_WIDE(lir->opcode)) { 1032 LIR* prev_insn = GetPrevEmittingLIR(lir); 1033 if (prev_insn == nullptr) { 1034 break; 1035 } 1036 uint64_t prev_insn_flags = EncodingMap[UNWIDE(prev_insn->opcode)].flags; 1037 // Check that the instruction preceding the multiply-accumulate is a load or store. 1038 if ((prev_insn_flags & IS_LOAD) != 0 || (prev_insn_flags & IS_STORE) != 0) { 1039 // insert a NOP between the load/store and the multiply-accumulate. 1040 LIR* new_lir = RawLIR(lir->dalvik_offset, kA64Nop0, 0, 0, 0, 0, 0, nullptr); 1041 new_lir->offset = lir->offset; 1042 new_lir->flags.fixup = kFixupNone; 1043 new_lir->flags.size = EncodingMap[kA64Nop0].size; 1044 InsertLIRBefore(lir, new_lir); 1045 lir->offset += new_lir->flags.size; 1046 offset_adjustment += new_lir->flags.size; 1047 res = kRetryAll; 1048 } 1049 } 1050 } 1051 break; 1052 default: 1053 LOG(FATAL) << "Unexpected case " << lir->flags.fixup; 1054 } 1055 prev_lir = lir; 1056 lir = lir->u.a.pcrel_next; 1057 } 1058 1059 if (res == kSuccess) { 1060 DCHECK_EQ(offset_adjustment, 0); 1061 break; 1062 } else { 1063 assembler_retries++; 1064 if (assembler_retries > MAX_ASSEMBLER_RETRIES) { 1065 CodegenDump(); 1066 LOG(FATAL) << "Assembler error - too many retries"; 1067 } 1068 starting_offset += offset_adjustment; 1069 data_offset_ = ALIGNED_DATA_OFFSET(starting_offset); 1070 AssignDataOffsets(); 1071 } 1072 } 1073 1074 // Build the CodeBuffer. 1075 DCHECK_LE(data_offset_, total_size_); 1076 code_buffer_.reserve(total_size_); 1077 code_buffer_.resize(starting_offset); 1078 uint8_t* write_pos = &code_buffer_[0]; 1079 write_pos = EncodeLIRs(write_pos, first_lir_insn_); 1080 DCHECK_EQ(static_cast<CodeOffset>(write_pos - &code_buffer_[0]), starting_offset); 1081 1082 DCHECK_EQ(data_offset_, ALIGNED_DATA_OFFSET(code_buffer_.size())); 1083 1084 // Install literals 1085 InstallLiteralPools(); 1086 1087 // Install switch tables 1088 InstallSwitchTables(); 1089 1090 // Install fill array data 1091 InstallFillArrayData(); 1092 1093 // Create the mapping table and native offset to reference map. 1094 cu_->NewTimingSplit("PcMappingTable"); 1095 CreateMappingTables(); 1096 1097 cu_->NewTimingSplit("GcMap"); 1098 CreateNativeGcMap(); 1099 } 1100 1101 size_t Arm64Mir2Lir::GetInsnSize(LIR* lir) { 1102 A64Opcode opcode = UNWIDE(lir->opcode); 1103 DCHECK(!IsPseudoLirOp(opcode)); 1104 return EncodingMap[opcode].size; 1105 } 1106 1107 // Encode instruction bit pattern and assign offsets. 1108 uint32_t Arm64Mir2Lir::LinkFixupInsns(LIR* head_lir, LIR* tail_lir, uint32_t offset) { 1109 LIR* end_lir = tail_lir->next; 1110 1111 LIR* last_fixup = nullptr; 1112 for (LIR* lir = head_lir; lir != end_lir; lir = NEXT_LIR(lir)) { 1113 A64Opcode opcode = UNWIDE(lir->opcode); 1114 if (!lir->flags.is_nop) { 1115 if (lir->flags.fixup != kFixupNone) { 1116 if (!IsPseudoLirOp(opcode)) { 1117 lir->flags.size = EncodingMap[opcode].size; 1118 lir->flags.fixup = EncodingMap[opcode].fixup; 1119 } else { 1120 DCHECK_NE(static_cast<int>(opcode), kPseudoPseudoAlign4); 1121 lir->flags.size = 0; 1122 lir->flags.fixup = kFixupLabel; 1123 } 1124 // Link into the fixup chain. 1125 lir->flags.use_def_invalid = true; 1126 lir->u.a.pcrel_next = nullptr; 1127 if (first_fixup_ == nullptr) { 1128 first_fixup_ = lir; 1129 } else { 1130 last_fixup->u.a.pcrel_next = lir; 1131 } 1132 last_fixup = lir; 1133 lir->offset = offset; 1134 } 1135 offset += lir->flags.size; 1136 } 1137 } 1138 return offset; 1139 } 1140 1141 void Arm64Mir2Lir::AssignDataOffsets() { 1142 /* Set up offsets for literals */ 1143 CodeOffset offset = data_offset_; 1144 1145 offset = AssignLiteralOffset(offset); 1146 1147 offset = AssignSwitchTablesOffset(offset); 1148 1149 total_size_ = AssignFillArrayDataOffset(offset); 1150 } 1151 1152 } // namespace art 1153
