Home | History | Annotate | Download | only in optimizing 
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include "code_generator_arm_vixl.h"
     18 #include "mirror/array-inl.h"
     19 
     20 namespace vixl32 = vixl::aarch32;
     21 using namespace vixl32;  // NOLINT(build/namespaces)
     22 
     23 namespace art {
     24 namespace arm {
     25 
     26 using helpers::DRegisterFrom;
     27 using helpers::Int64ConstantFrom;
     28 using helpers::InputDRegisterAt;
     29 using helpers::InputRegisterAt;
     30 using helpers::OutputDRegister;
     31 using helpers::OutputRegister;
     32 using helpers::RegisterFrom;
     33 
     34 #define __ GetVIXLAssembler()->
     35 
     36 void LocationsBuilderARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
     37   LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
     38   switch (instruction->GetPackedType()) {
     39     case DataType::Type::kBool:
     40     case DataType::Type::kUint8:
     41     case DataType::Type::kInt8:
     42     case DataType::Type::kUint16:
     43     case DataType::Type::kInt16:
     44     case DataType::Type::kInt32:
     45       locations->SetInAt(0, Location::RequiresRegister());
     46       locations->SetOut(Location::RequiresFpuRegister());
     47       break;
     48     default:
     49       LOG(FATAL) << "Unsupported SIMD type";
     50       UNREACHABLE();
     51   }
     52 }
     53 
     54 void InstructionCodeGeneratorARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
     55   LocationSummary* locations = instruction->GetLocations();
     56   vixl32::DRegister dst = DRegisterFrom(locations->Out());
     57   switch (instruction->GetPackedType()) {
     58     case DataType::Type::kBool:
     59     case DataType::Type::kUint8:
     60     case DataType::Type::kInt8:
     61       DCHECK_EQ(8u, instruction->GetVectorLength());
     62       __ Vdup(Untyped8, dst, InputRegisterAt(instruction, 0));
     63       break;
     64     case DataType::Type::kUint16:
     65     case DataType::Type::kInt16:
     66       DCHECK_EQ(4u, instruction->GetVectorLength());
     67       __ Vdup(Untyped16, dst, InputRegisterAt(instruction, 0));
     68       break;
     69     case DataType::Type::kInt32:
     70       DCHECK_EQ(2u, instruction->GetVectorLength());
     71       __ Vdup(Untyped32, dst, InputRegisterAt(instruction, 0));
     72       break;
     73     default:
     74       LOG(FATAL) << "Unsupported SIMD type";
     75       UNREACHABLE();
     76   }
     77 }
     78 
     79 void LocationsBuilderARMVIXL::VisitVecExtractScalar(HVecExtractScalar* instruction) {
     80   LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
     81   switch (instruction->GetPackedType()) {
     82     case DataType::Type::kInt32:
     83       locations->SetInAt(0, Location::RequiresFpuRegister());
     84       locations->SetOut(Location::RequiresRegister());
     85       break;
     86     default:
     87       LOG(FATAL) << "Unsupported SIMD type";
     88       UNREACHABLE();
     89   }
     90 }
     91 
     92 void InstructionCodeGeneratorARMVIXL::VisitVecExtractScalar(HVecExtractScalar* instruction) {
     93   LocationSummary* locations = instruction->GetLocations();
     94   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
     95   switch (instruction->GetPackedType()) {
     96     case DataType::Type::kInt32:
     97       DCHECK_EQ(2u, instruction->GetVectorLength());
     98       __ Vmov(OutputRegister(instruction), DRegisterLane(src, 0));
     99       break;
    100     default:
    101       LOG(FATAL) << "Unsupported SIMD type";
    102       UNREACHABLE();
    103   }
    104 }
    105 
    106 // Helper to set up locations for vector unary operations.
    107 static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation* instruction) {
    108   LocationSummary* locations = new (allocator) LocationSummary(instruction);
    109   switch (instruction->GetPackedType()) {
    110     case DataType::Type::kBool:
    111       locations->SetInAt(0, Location::RequiresFpuRegister());
    112       locations->SetOut(Location::RequiresFpuRegister(),
    113                         instruction->IsVecNot() ? Location::kOutputOverlap
    114                                                 : Location::kNoOutputOverlap);
    115       break;
    116     case DataType::Type::kUint8:
    117     case DataType::Type::kInt8:
    118     case DataType::Type::kUint16:
    119     case DataType::Type::kInt16:
    120     case DataType::Type::kInt32:
    121       locations->SetInAt(0, Location::RequiresFpuRegister());
    122       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
    123       break;
    124     default:
    125       LOG(FATAL) << "Unsupported SIMD type";
    126       UNREACHABLE();
    127   }
    128 }
    129 
    130 void LocationsBuilderARMVIXL::VisitVecReduce(HVecReduce* instruction) {
    131   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
    132 }
    133 
    134 void InstructionCodeGeneratorARMVIXL::VisitVecReduce(HVecReduce* instruction) {
    135   LocationSummary* locations = instruction->GetLocations();
    136   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
    137   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    138   switch (instruction->GetPackedType()) {
    139     case DataType::Type::kInt32:
    140       DCHECK_EQ(2u, instruction->GetVectorLength());
    141       switch (instruction->GetKind()) {
    142         case HVecReduce::kSum:
    143           __ Vpadd(DataTypeValue::I32, dst, src, src);
    144           break;
    145         case HVecReduce::kMin:
    146           __ Vpmin(DataTypeValue::S32, dst, src, src);
    147           break;
    148         case HVecReduce::kMax:
    149           __ Vpmax(DataTypeValue::S32, dst, src, src);
    150           break;
    151       }
    152       break;
    153     default:
    154       LOG(FATAL) << "Unsupported SIMD type";
    155       UNREACHABLE();
    156   }
    157 }
    158 
    159 void LocationsBuilderARMVIXL::VisitVecCnv(HVecCnv* instruction) {
    160   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
    161 }
    162 
    163 void InstructionCodeGeneratorARMVIXL::VisitVecCnv(HVecCnv* instruction) {
    164   LOG(FATAL) << "No SIMD for " << instruction->GetId();
    165 }
    166 
    167 void LocationsBuilderARMVIXL::VisitVecNeg(HVecNeg* instruction) {
    168   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
    169 }
    170 
    171 void InstructionCodeGeneratorARMVIXL::VisitVecNeg(HVecNeg* instruction) {
    172   LocationSummary* locations = instruction->GetLocations();
    173   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
    174   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    175   switch (instruction->GetPackedType()) {
    176     case DataType::Type::kUint8:
    177     case DataType::Type::kInt8:
    178       DCHECK_EQ(8u, instruction->GetVectorLength());
    179       __ Vneg(DataTypeValue::S8, dst, src);
    180       break;
    181     case DataType::Type::kUint16:
    182     case DataType::Type::kInt16:
    183       DCHECK_EQ(4u, instruction->GetVectorLength());
    184       __ Vneg(DataTypeValue::S16, dst, src);
    185       break;
    186     case DataType::Type::kInt32:
    187       DCHECK_EQ(2u, instruction->GetVectorLength());
    188       __ Vneg(DataTypeValue::S32, dst, src);
    189       break;
    190     default:
    191       LOG(FATAL) << "Unsupported SIMD type";
    192       UNREACHABLE();
    193   }
    194 }
    195 
    196 void LocationsBuilderARMVIXL::VisitVecAbs(HVecAbs* instruction) {
    197   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
    198 }
    199 
    200 void InstructionCodeGeneratorARMVIXL::VisitVecAbs(HVecAbs* instruction) {
    201   LocationSummary* locations = instruction->GetLocations();
    202   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
    203   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    204   switch (instruction->GetPackedType()) {
    205     case DataType::Type::kInt8:
    206       DCHECK_EQ(8u, instruction->GetVectorLength());
    207       __ Vabs(DataTypeValue::S8, dst, src);
    208       break;
    209     case DataType::Type::kInt16:
    210       DCHECK_EQ(4u, instruction->GetVectorLength());
    211       __ Vabs(DataTypeValue::S16, dst, src);
    212       break;
    213     case DataType::Type::kInt32:
    214       DCHECK_EQ(2u, instruction->GetVectorLength());
    215       __ Vabs(DataTypeValue::S32, dst, src);
    216       break;
    217     default:
    218       LOG(FATAL) << "Unsupported SIMD type";
    219       UNREACHABLE();
    220   }
    221 }
    222 
    223 void LocationsBuilderARMVIXL::VisitVecNot(HVecNot* instruction) {
    224   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
    225 }
    226 
    227 void InstructionCodeGeneratorARMVIXL::VisitVecNot(HVecNot* instruction) {
    228   LocationSummary* locations = instruction->GetLocations();
    229   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
    230   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    231   switch (instruction->GetPackedType()) {
    232     case DataType::Type::kBool:  // special case boolean-not
    233       DCHECK_EQ(8u, instruction->GetVectorLength());
    234       __ Vmov(I8, dst, 1);
    235       __ Veor(dst, dst, src);
    236       break;
    237     case DataType::Type::kUint8:
    238     case DataType::Type::kInt8:
    239     case DataType::Type::kUint16:
    240     case DataType::Type::kInt16:
    241     case DataType::Type::kInt32:
    242       __ Vmvn(I8, dst, src);  // lanes do not matter
    243       break;
    244     default:
    245       LOG(FATAL) << "Unsupported SIMD type";
    246       UNREACHABLE();
    247   }
    248 }
    249 
    250 // Helper to set up locations for vector binary operations.
    251 static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
    252   LocationSummary* locations = new (allocator) LocationSummary(instruction);
    253   switch (instruction->GetPackedType()) {
    254     case DataType::Type::kBool:
    255     case DataType::Type::kUint8:
    256     case DataType::Type::kInt8:
    257     case DataType::Type::kUint16:
    258     case DataType::Type::kInt16:
    259     case DataType::Type::kInt32:
    260       locations->SetInAt(0, Location::RequiresFpuRegister());
    261       locations->SetInAt(1, Location::RequiresFpuRegister());
    262       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
    263       break;
    264     default:
    265       LOG(FATAL) << "Unsupported SIMD type";
    266       UNREACHABLE();
    267   }
    268 }
    269 
    270 void LocationsBuilderARMVIXL::VisitVecAdd(HVecAdd* instruction) {
    271   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    272 }
    273 
    274 void InstructionCodeGeneratorARMVIXL::VisitVecAdd(HVecAdd* instruction) {
    275   LocationSummary* locations = instruction->GetLocations();
    276   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    277   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    278   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    279   switch (instruction->GetPackedType()) {
    280     case DataType::Type::kUint8:
    281     case DataType::Type::kInt8:
    282       DCHECK_EQ(8u, instruction->GetVectorLength());
    283       __ Vadd(I8, dst, lhs, rhs);
    284       break;
    285     case DataType::Type::kUint16:
    286     case DataType::Type::kInt16:
    287       DCHECK_EQ(4u, instruction->GetVectorLength());
    288       __ Vadd(I16, dst, lhs, rhs);
    289       break;
    290     case DataType::Type::kInt32:
    291       DCHECK_EQ(2u, instruction->GetVectorLength());
    292       __ Vadd(I32, dst, lhs, rhs);
    293       break;
    294     default:
    295       LOG(FATAL) << "Unsupported SIMD type";
    296       UNREACHABLE();
    297   }
    298 }
    299 
    300 void LocationsBuilderARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
    301   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    302 }
    303 
    304 void InstructionCodeGeneratorARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
    305   LocationSummary* locations = instruction->GetLocations();
    306   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    307   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    308   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    309   switch (instruction->GetPackedType()) {
    310     case DataType::Type::kUint8:
    311       DCHECK_EQ(8u, instruction->GetVectorLength());
    312       instruction->IsRounded()
    313           ? __ Vrhadd(DataTypeValue::U8, dst, lhs, rhs)
    314           : __ Vhadd(DataTypeValue::U8, dst, lhs, rhs);
    315       break;
    316     case DataType::Type::kInt8:
    317       DCHECK_EQ(8u, instruction->GetVectorLength());
    318       instruction->IsRounded()
    319           ? __ Vrhadd(DataTypeValue::S8, dst, lhs, rhs)
    320           : __ Vhadd(DataTypeValue::S8, dst, lhs, rhs);
    321       break;
    322     case DataType::Type::kUint16:
    323       DCHECK_EQ(4u, instruction->GetVectorLength());
    324       instruction->IsRounded()
    325           ? __ Vrhadd(DataTypeValue::U16, dst, lhs, rhs)
    326           : __ Vhadd(DataTypeValue::U16, dst, lhs, rhs);
    327       break;
    328     case DataType::Type::kInt16:
    329       DCHECK_EQ(4u, instruction->GetVectorLength());
    330       instruction->IsRounded()
    331           ? __ Vrhadd(DataTypeValue::S16, dst, lhs, rhs)
    332           : __ Vhadd(DataTypeValue::S16, dst, lhs, rhs);
    333       break;
    334     default:
    335       LOG(FATAL) << "Unsupported SIMD type";
    336       UNREACHABLE();
    337   }
    338 }
    339 
    340 void LocationsBuilderARMVIXL::VisitVecSub(HVecSub* instruction) {
    341   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    342 }
    343 
    344 void InstructionCodeGeneratorARMVIXL::VisitVecSub(HVecSub* instruction) {
    345   LocationSummary* locations = instruction->GetLocations();
    346   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    347   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    348   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    349   switch (instruction->GetPackedType()) {
    350     case DataType::Type::kUint8:
    351     case DataType::Type::kInt8:
    352       DCHECK_EQ(8u, instruction->GetVectorLength());
    353       __ Vsub(I8, dst, lhs, rhs);
    354       break;
    355     case DataType::Type::kUint16:
    356     case DataType::Type::kInt16:
    357       DCHECK_EQ(4u, instruction->GetVectorLength());
    358       __ Vsub(I16, dst, lhs, rhs);
    359       break;
    360     case DataType::Type::kInt32:
    361       DCHECK_EQ(2u, instruction->GetVectorLength());
    362       __ Vsub(I32, dst, lhs, rhs);
    363       break;
    364     default:
    365       LOG(FATAL) << "Unsupported SIMD type";
    366       UNREACHABLE();
    367   }
    368 }
    369 
    370 void LocationsBuilderARMVIXL::VisitVecMul(HVecMul* instruction) {
    371   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    372 }
    373 
    374 void InstructionCodeGeneratorARMVIXL::VisitVecMul(HVecMul* instruction) {
    375   LocationSummary* locations = instruction->GetLocations();
    376   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    377   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    378   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    379   switch (instruction->GetPackedType()) {
    380     case DataType::Type::kUint8:
    381     case DataType::Type::kInt8:
    382       DCHECK_EQ(8u, instruction->GetVectorLength());
    383       __ Vmul(I8, dst, lhs, rhs);
    384       break;
    385     case DataType::Type::kUint16:
    386     case DataType::Type::kInt16:
    387       DCHECK_EQ(4u, instruction->GetVectorLength());
    388       __ Vmul(I16, dst, lhs, rhs);
    389       break;
    390     case DataType::Type::kInt32:
    391       DCHECK_EQ(2u, instruction->GetVectorLength());
    392       __ Vmul(I32, dst, lhs, rhs);
    393       break;
    394     default:
    395       LOG(FATAL) << "Unsupported SIMD type";
    396       UNREACHABLE();
    397   }
    398 }
    399 
    400 void LocationsBuilderARMVIXL::VisitVecDiv(HVecDiv* instruction) {
    401   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    402 }
    403 
    404 void InstructionCodeGeneratorARMVIXL::VisitVecDiv(HVecDiv* instruction) {
    405   LOG(FATAL) << "No SIMD for " << instruction->GetId();
    406 }
    407 
    408 void LocationsBuilderARMVIXL::VisitVecMin(HVecMin* instruction) {
    409   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    410 }
    411 
    412 void InstructionCodeGeneratorARMVIXL::VisitVecMin(HVecMin* instruction) {
    413   LocationSummary* locations = instruction->GetLocations();
    414   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    415   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    416   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    417   switch (instruction->GetPackedType()) {
    418     case DataType::Type::kUint8:
    419       DCHECK_EQ(8u, instruction->GetVectorLength());
    420       __ Vmin(DataTypeValue::U8, dst, lhs, rhs);
    421       break;
    422     case DataType::Type::kInt8:
    423       DCHECK_EQ(8u, instruction->GetVectorLength());
    424       __ Vmin(DataTypeValue::S8, dst, lhs, rhs);
    425       break;
    426     case DataType::Type::kUint16:
    427       DCHECK_EQ(4u, instruction->GetVectorLength());
    428       __ Vmin(DataTypeValue::U16, dst, lhs, rhs);
    429       break;
    430     case DataType::Type::kInt16:
    431       DCHECK_EQ(4u, instruction->GetVectorLength());
    432       __ Vmin(DataTypeValue::S16, dst, lhs, rhs);
    433       break;
    434     case DataType::Type::kUint32:
    435       DCHECK_EQ(2u, instruction->GetVectorLength());
    436       __ Vmin(DataTypeValue::U32, dst, lhs, rhs);
    437       break;
    438     case DataType::Type::kInt32:
    439       DCHECK_EQ(2u, instruction->GetVectorLength());
    440       __ Vmin(DataTypeValue::S32, dst, lhs, rhs);
    441       break;
    442     default:
    443       LOG(FATAL) << "Unsupported SIMD type";
    444       UNREACHABLE();
    445   }
    446 }
    447 
    448 void LocationsBuilderARMVIXL::VisitVecMax(HVecMax* instruction) {
    449   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    450 }
    451 
    452 void InstructionCodeGeneratorARMVIXL::VisitVecMax(HVecMax* instruction) {
    453   LocationSummary* locations = instruction->GetLocations();
    454   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    455   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    456   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    457   switch (instruction->GetPackedType()) {
    458     case DataType::Type::kUint8:
    459       DCHECK_EQ(8u, instruction->GetVectorLength());
    460       __ Vmax(DataTypeValue::U8, dst, lhs, rhs);
    461       break;
    462     case DataType::Type::kInt8:
    463       DCHECK_EQ(8u, instruction->GetVectorLength());
    464       __ Vmax(DataTypeValue::S8, dst, lhs, rhs);
    465       break;
    466     case DataType::Type::kUint16:
    467       DCHECK_EQ(4u, instruction->GetVectorLength());
    468       __ Vmax(DataTypeValue::U16, dst, lhs, rhs);
    469       break;
    470     case DataType::Type::kInt16:
    471       DCHECK_EQ(4u, instruction->GetVectorLength());
    472       __ Vmax(DataTypeValue::S16, dst, lhs, rhs);
    473       break;
    474     case DataType::Type::kUint32:
    475       DCHECK_EQ(2u, instruction->GetVectorLength());
    476       __ Vmax(DataTypeValue::U32, dst, lhs, rhs);
    477       break;
    478     case DataType::Type::kInt32:
    479       DCHECK_EQ(2u, instruction->GetVectorLength());
    480       __ Vmax(DataTypeValue::S32, dst, lhs, rhs);
    481       break;
    482     default:
    483       LOG(FATAL) << "Unsupported SIMD type";
    484       UNREACHABLE();
    485   }
    486 }
    487 
    488 void LocationsBuilderARMVIXL::VisitVecAnd(HVecAnd* instruction) {
    489   // TODO: Allow constants supported by VAND (immediate).
    490   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    491 }
    492 
    493 void InstructionCodeGeneratorARMVIXL::VisitVecAnd(HVecAnd* instruction) {
    494   LocationSummary* locations = instruction->GetLocations();
    495   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    496   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    497   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    498   switch (instruction->GetPackedType()) {
    499     case DataType::Type::kBool:
    500     case DataType::Type::kUint8:
    501     case DataType::Type::kInt8:
    502     case DataType::Type::kUint16:
    503     case DataType::Type::kInt16:
    504     case DataType::Type::kInt32:
    505       __ Vand(I8, dst, lhs, rhs);
    506       break;
    507     default:
    508       LOG(FATAL) << "Unsupported SIMD type";
    509       UNREACHABLE();
    510   }
    511 }
    512 
    513 void LocationsBuilderARMVIXL::VisitVecAndNot(HVecAndNot* instruction) {
    514   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    515 }
    516 
    517 void InstructionCodeGeneratorARMVIXL::VisitVecAndNot(HVecAndNot* instruction) {
    518   LOG(FATAL) << "No SIMD for " << instruction->GetId();
    519 }
    520 
    521 void LocationsBuilderARMVIXL::VisitVecOr(HVecOr* instruction) {
    522   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    523 }
    524 
    525 void InstructionCodeGeneratorARMVIXL::VisitVecOr(HVecOr* instruction) {
    526   LocationSummary* locations = instruction->GetLocations();
    527   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    528   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    529   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    530   switch (instruction->GetPackedType()) {
    531     case DataType::Type::kBool:
    532     case DataType::Type::kUint8:
    533     case DataType::Type::kInt8:
    534     case DataType::Type::kUint16:
    535     case DataType::Type::kInt16:
    536     case DataType::Type::kInt32:
    537       __ Vorr(I8, dst, lhs, rhs);
    538       break;
    539     default:
    540       LOG(FATAL) << "Unsupported SIMD type";
    541       UNREACHABLE();
    542   }
    543 }
    544 
    545 void LocationsBuilderARMVIXL::VisitVecXor(HVecXor* instruction) {
    546   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
    547 }
    548 
    549 void InstructionCodeGeneratorARMVIXL::VisitVecXor(HVecXor* instruction) {
    550   LocationSummary* locations = instruction->GetLocations();
    551   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    552   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
    553   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    554   switch (instruction->GetPackedType()) {
    555     case DataType::Type::kBool:
    556     case DataType::Type::kUint8:
    557     case DataType::Type::kInt8:
    558     case DataType::Type::kUint16:
    559     case DataType::Type::kInt16:
    560     case DataType::Type::kInt32:
    561       __ Veor(I8, dst, lhs, rhs);
    562       break;
    563     default:
    564       LOG(FATAL) << "Unsupported SIMD type";
    565       UNREACHABLE();
    566   }
    567 }
    568 
    569 // Helper to set up locations for vector shift operations.
    570 static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
    571   LocationSummary* locations = new (allocator) LocationSummary(instruction);
    572   switch (instruction->GetPackedType()) {
    573     case DataType::Type::kUint8:
    574     case DataType::Type::kInt8:
    575     case DataType::Type::kUint16:
    576     case DataType::Type::kInt16:
    577     case DataType::Type::kInt32:
    578       locations->SetInAt(0, Location::RequiresFpuRegister());
    579       locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant()));
    580       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
    581       break;
    582     default:
    583       LOG(FATAL) << "Unsupported SIMD type";
    584       UNREACHABLE();
    585   }
    586 }
    587 
    588 void LocationsBuilderARMVIXL::VisitVecShl(HVecShl* instruction) {
    589   CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
    590 }
    591 
    592 void InstructionCodeGeneratorARMVIXL::VisitVecShl(HVecShl* instruction) {
    593   LocationSummary* locations = instruction->GetLocations();
    594   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    595   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    596   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
    597   switch (instruction->GetPackedType()) {
    598     case DataType::Type::kUint8:
    599     case DataType::Type::kInt8:
    600       DCHECK_EQ(8u, instruction->GetVectorLength());
    601       __ Vshl(I8, dst, lhs, value);
    602       break;
    603     case DataType::Type::kUint16:
    604     case DataType::Type::kInt16:
    605       DCHECK_EQ(4u, instruction->GetVectorLength());
    606       __ Vshl(I16, dst, lhs, value);
    607       break;
    608     case DataType::Type::kInt32:
    609       DCHECK_EQ(2u, instruction->GetVectorLength());
    610       __ Vshl(I32, dst, lhs, value);
    611       break;
    612     default:
    613       LOG(FATAL) << "Unsupported SIMD type";
    614       UNREACHABLE();
    615   }
    616 }
    617 
    618 void LocationsBuilderARMVIXL::VisitVecShr(HVecShr* instruction) {
    619   CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
    620 }
    621 
    622 void InstructionCodeGeneratorARMVIXL::VisitVecShr(HVecShr* instruction) {
    623   LocationSummary* locations = instruction->GetLocations();
    624   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    625   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    626   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
    627   switch (instruction->GetPackedType()) {
    628     case DataType::Type::kUint8:
    629     case DataType::Type::kInt8:
    630       DCHECK_EQ(8u, instruction->GetVectorLength());
    631       __ Vshr(DataTypeValue::S8, dst, lhs, value);
    632       break;
    633     case DataType::Type::kUint16:
    634     case DataType::Type::kInt16:
    635       DCHECK_EQ(4u, instruction->GetVectorLength());
    636       __ Vshr(DataTypeValue::S16, dst, lhs, value);
    637       break;
    638     case DataType::Type::kInt32:
    639       DCHECK_EQ(2u, instruction->GetVectorLength());
    640       __ Vshr(DataTypeValue::S32, dst, lhs, value);
    641       break;
    642     default:
    643       LOG(FATAL) << "Unsupported SIMD type";
    644       UNREACHABLE();
    645   }
    646 }
    647 
    648 void LocationsBuilderARMVIXL::VisitVecUShr(HVecUShr* instruction) {
    649   CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
    650 }
    651 
    652 void InstructionCodeGeneratorARMVIXL::VisitVecUShr(HVecUShr* instruction) {
    653   LocationSummary* locations = instruction->GetLocations();
    654   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
    655   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    656   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
    657   switch (instruction->GetPackedType()) {
    658     case DataType::Type::kUint8:
    659     case DataType::Type::kInt8:
    660       DCHECK_EQ(8u, instruction->GetVectorLength());
    661       __ Vshr(DataTypeValue::U8, dst, lhs, value);
    662       break;
    663     case DataType::Type::kUint16:
    664     case DataType::Type::kInt16:
    665       DCHECK_EQ(4u, instruction->GetVectorLength());
    666       __ Vshr(DataTypeValue::U16, dst, lhs, value);
    667       break;
    668     case DataType::Type::kInt32:
    669       DCHECK_EQ(2u, instruction->GetVectorLength());
    670       __ Vshr(DataTypeValue::U32, dst, lhs, value);
    671       break;
    672     default:
    673       LOG(FATAL) << "Unsupported SIMD type";
    674       UNREACHABLE();
    675   }
    676 }
    677 
    678 void LocationsBuilderARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) {
    679   LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
    680 
    681   DCHECK_EQ(1u, instruction->InputCount());  // only one input currently implemented
    682 
    683   HInstruction* input = instruction->InputAt(0);
    684   bool is_zero = IsZeroBitPattern(input);
    685 
    686   switch (instruction->GetPackedType()) {
    687     case DataType::Type::kInt32:
    688       locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant())
    689                                     : Location::RequiresRegister());
    690       locations->SetOut(Location::RequiresFpuRegister());
    691       break;
    692     default:
    693       LOG(FATAL) << "Unsupported SIMD type";
    694       UNREACHABLE();
    695   }
    696 }
    697 
    698 void InstructionCodeGeneratorARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) {
    699   LocationSummary* locations = instruction->GetLocations();
    700   vixl32::DRegister dst = DRegisterFrom(locations->Out());
    701 
    702   DCHECK_EQ(1u, instruction->InputCount());  // only one input currently implemented
    703 
    704   // Zero out all other elements first.
    705   __ Vmov(I32, dst, 0);
    706 
    707   // Shorthand for any type of zero.
    708   if (IsZeroBitPattern(instruction->InputAt(0))) {
    709     return;
    710   }
    711 
    712   // Set required elements.
    713   switch (instruction->GetPackedType()) {
    714     case DataType::Type::kInt32:
    715       DCHECK_EQ(2u, instruction->GetVectorLength());
    716       __ Vmov(Untyped32, DRegisterLane(dst, 0), InputRegisterAt(instruction, 0));
    717       break;
    718     default:
    719       LOG(FATAL) << "Unsupported SIMD type";
    720       UNREACHABLE();
    721   }
    722 }
    723 
    724 // Helper to set up locations for vector accumulations.
    725 static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* instruction) {
    726   LocationSummary* locations = new (allocator) LocationSummary(instruction);
    727   switch (instruction->GetPackedType()) {
    728     case DataType::Type::kUint8:
    729     case DataType::Type::kInt8:
    730     case DataType::Type::kUint16:
    731     case DataType::Type::kInt16:
    732     case DataType::Type::kInt32:
    733     case DataType::Type::kInt64:
    734       locations->SetInAt(0, Location::RequiresFpuRegister());
    735       locations->SetInAt(1, Location::RequiresFpuRegister());
    736       locations->SetInAt(2, Location::RequiresFpuRegister());
    737       locations->SetOut(Location::SameAsFirstInput());
    738       break;
    739     default:
    740       LOG(FATAL) << "Unsupported SIMD type";
    741       UNREACHABLE();
    742   }
    743 }
    744 
    745 void LocationsBuilderARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
    746   CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
    747 }
    748 
    749 void InstructionCodeGeneratorARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
    750   LOG(FATAL) << "No SIMD for " << instruction->GetId();
    751 }
    752 
    753 void LocationsBuilderARMVIXL::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
    754   CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
    755 }
    756 
    757 void InstructionCodeGeneratorARMVIXL::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
    758   LocationSummary* locations = instruction->GetLocations();
    759   vixl32::DRegister acc = DRegisterFrom(locations->InAt(0));
    760   vixl32::DRegister left = DRegisterFrom(locations->InAt(1));
    761   vixl32::DRegister right = DRegisterFrom(locations->InAt(2));
    762 
    763   DCHECK(locations->InAt(0).Equals(locations->Out()));
    764 
    765   // Handle all feasible acc_T += sad(a_S, b_S) type combinations (T x S).
    766   HVecOperation* a = instruction->InputAt(1)->AsVecOperation();
    767   HVecOperation* b = instruction->InputAt(2)->AsVecOperation();
    768   DCHECK_EQ(a->GetPackedType(), b->GetPackedType());
    769   switch (a->GetPackedType()) {
    770     case DataType::Type::kInt32:
    771       DCHECK_EQ(2u, a->GetVectorLength());
    772       switch (instruction->GetPackedType()) {
    773         case DataType::Type::kInt32: {
    774           DCHECK_EQ(2u, instruction->GetVectorLength());
    775           UseScratchRegisterScope temps(GetVIXLAssembler());
    776           vixl32::DRegister tmp = temps.AcquireD();
    777           __ Vsub(DataTypeValue::I32, tmp, left, right);
    778           __ Vabs(DataTypeValue::S32, tmp, tmp);
    779           __ Vadd(DataTypeValue::I32, acc, acc, tmp);
    780           break;
    781         }
    782         default:
    783           LOG(FATAL) << "Unsupported SIMD type";
    784           UNREACHABLE();
    785       }
    786       break;
    787     default:
    788       LOG(FATAL) << "Unsupported SIMD type";
    789       UNREACHABLE();
    790   }
    791 }
    792 
    793 // Return whether the vector memory access operation is guaranteed to be word-aligned (ARM word
    794 // size equals to 4).
    795 static bool IsWordAligned(HVecMemoryOperation* instruction) {
    796   return instruction->GetAlignment().IsAlignedAt(4u);
    797 }
    798 
    799 // Helper to set up locations for vector memory operations.
    800 static void CreateVecMemLocations(ArenaAllocator* allocator,
    801                                   HVecMemoryOperation* instruction,
    802                                   bool is_load) {
    803   LocationSummary* locations = new (allocator) LocationSummary(instruction);
    804   switch (instruction->GetPackedType()) {
    805     case DataType::Type::kBool:
    806     case DataType::Type::kUint8:
    807     case DataType::Type::kInt8:
    808     case DataType::Type::kUint16:
    809     case DataType::Type::kInt16:
    810     case DataType::Type::kInt32:
    811       locations->SetInAt(0, Location::RequiresRegister());
    812       locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
    813       if (is_load) {
    814         locations->SetOut(Location::RequiresFpuRegister());
    815       } else {
    816         locations->SetInAt(2, Location::RequiresFpuRegister());
    817       }
    818       break;
    819     default:
    820       LOG(FATAL) << "Unsupported SIMD type";
    821       UNREACHABLE();
    822   }
    823 }
    824 
    825 // Helper to set up locations for vector memory operations. Returns the memory operand and,
    826 // if used, sets the output parameter scratch to a temporary register used in this operand,
    827 // so that the client can release it right after the memory operand use.
    828 MemOperand InstructionCodeGeneratorARMVIXL::VecAddress(
    829         HVecMemoryOperation* instruction,
    830         UseScratchRegisterScope* temps_scope,
    831         /*out*/ vixl32::Register* scratch) {
    832   LocationSummary* locations = instruction->GetLocations();
    833   vixl32::Register base = InputRegisterAt(instruction, 0);
    834 
    835   Location index = locations->InAt(1);
    836   size_t size = DataType::Size(instruction->GetPackedType());
    837   uint32_t offset = mirror::Array::DataOffset(size).Uint32Value();
    838   size_t shift = ComponentSizeShiftWidth(size);
    839 
    840   // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet.
    841   DCHECK(!instruction->InputAt(0)->IsIntermediateAddress());
    842 
    843   if (index.IsConstant()) {
    844     offset += Int64ConstantFrom(index) << shift;
    845     return MemOperand(base, offset);
    846   } else {
    847     *scratch = temps_scope->Acquire();
    848     __ Add(*scratch, base, Operand(RegisterFrom(index), ShiftType::LSL, shift));
    849 
    850     return MemOperand(*scratch, offset);
    851   }
    852 }
    853 
    854 AlignedMemOperand InstructionCodeGeneratorARMVIXL::VecAddressUnaligned(
    855         HVecMemoryOperation* instruction,
    856         UseScratchRegisterScope* temps_scope,
    857         /*out*/ vixl32::Register* scratch) {
    858   LocationSummary* locations = instruction->GetLocations();
    859   vixl32::Register base = InputRegisterAt(instruction, 0);
    860 
    861   Location index = locations->InAt(1);
    862   size_t size = DataType::Size(instruction->GetPackedType());
    863   uint32_t offset = mirror::Array::DataOffset(size).Uint32Value();
    864   size_t shift = ComponentSizeShiftWidth(size);
    865 
    866   // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet.
    867   DCHECK(!instruction->InputAt(0)->IsIntermediateAddress());
    868 
    869   if (index.IsConstant()) {
    870     offset += Int64ConstantFrom(index) << shift;
    871     __ Add(*scratch, base, offset);
    872   } else {
    873     *scratch = temps_scope->Acquire();
    874     __ Add(*scratch, base, offset);
    875     __ Add(*scratch, *scratch, Operand(RegisterFrom(index), ShiftType::LSL, shift));
    876   }
    877   return AlignedMemOperand(*scratch, kNoAlignment);
    878 }
    879 
    880 void LocationsBuilderARMVIXL::VisitVecLoad(HVecLoad* instruction) {
    881   CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ true);
    882 }
    883 
    884 void InstructionCodeGeneratorARMVIXL::VisitVecLoad(HVecLoad* instruction) {
    885   vixl32::DRegister reg = OutputDRegister(instruction);
    886   UseScratchRegisterScope temps(GetVIXLAssembler());
    887   vixl32::Register scratch;
    888 
    889   DCHECK(instruction->GetPackedType() != DataType::Type::kUint16 || !instruction->IsStringCharAt());
    890 
    891   switch (instruction->GetPackedType()) {
    892     case DataType::Type::kBool:
    893     case DataType::Type::kUint8:
    894     case DataType::Type::kInt8:
    895       DCHECK_EQ(8u, instruction->GetVectorLength());
    896       if (IsWordAligned(instruction)) {
    897         __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
    898       } else {
    899         __ Vld1(Untyped8,
    900             NeonRegisterList(reg, kMultipleLanes),
    901             VecAddressUnaligned(instruction, &temps, &scratch));
    902       }
    903       break;
    904     case DataType::Type::kUint16:
    905     case DataType::Type::kInt16:
    906       DCHECK_EQ(4u, instruction->GetVectorLength());
    907       if (IsWordAligned(instruction)) {
    908         __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
    909       } else {
    910         __ Vld1(Untyped16,
    911             NeonRegisterList(reg, kMultipleLanes),
    912             VecAddressUnaligned(instruction, &temps, &scratch));
    913       }
    914       break;
    915     case DataType::Type::kInt32:
    916       DCHECK_EQ(2u, instruction->GetVectorLength());
    917       if (IsWordAligned(instruction)) {
    918         __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
    919       } else {
    920         __ Vld1(Untyped32,
    921             NeonRegisterList(reg, kMultipleLanes),
    922             VecAddressUnaligned(instruction, &temps, &scratch));
    923       }
    924       break;
    925     default:
    926       LOG(FATAL) << "Unsupported SIMD type";
    927       UNREACHABLE();
    928   }
    929 }
    930 
    931 void LocationsBuilderARMVIXL::VisitVecStore(HVecStore* instruction) {
    932   CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ false);
    933 }
    934 
    935 void InstructionCodeGeneratorARMVIXL::VisitVecStore(HVecStore* instruction) {
    936   vixl32::DRegister reg = InputDRegisterAt(instruction, 2);
    937   UseScratchRegisterScope temps(GetVIXLAssembler());
    938   vixl32::Register scratch;
    939   switch (instruction->GetPackedType()) {
    940     case DataType::Type::kBool:
    941     case DataType::Type::kUint8:
    942     case DataType::Type::kInt8:
    943       DCHECK_EQ(8u, instruction->GetVectorLength());
    944       if (IsWordAligned(instruction)) {
    945         __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
    946       } else {
    947         __ Vst1(Untyped8,
    948                 NeonRegisterList(reg, kMultipleLanes),
    949                 VecAddressUnaligned(instruction, &temps, &scratch));
    950       }
    951       break;
    952     case DataType::Type::kUint16:
    953     case DataType::Type::kInt16:
    954       DCHECK_EQ(4u, instruction->GetVectorLength());
    955       if (IsWordAligned(instruction)) {
    956         __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
    957       } else {
    958         __ Vst1(Untyped16,
    959                 NeonRegisterList(reg, kMultipleLanes),
    960                 VecAddressUnaligned(instruction, &temps, &scratch));
    961       }
    962       break;
    963     case DataType::Type::kInt32:
    964       DCHECK_EQ(2u, instruction->GetVectorLength());
    965       if (IsWordAligned(instruction)) {
    966         __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
    967       } else {
    968         __ Vst1(Untyped32,
    969                 NeonRegisterList(reg, kMultipleLanes),
    970                 VecAddressUnaligned(instruction, &temps, &scratch));
    971       }
    972       break;
    973     default:
    974       LOG(FATAL) << "Unsupported SIMD type";
    975       UNREACHABLE();
    976   }
    977 }
    978 
    979 #undef __
    980 
    981 }  // namespace arm
    982 }  // namespace art
    983