Home | History | Annotate | Download | only in optimizing
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #ifndef ART_COMPILER_OPTIMIZING_NODES_VECTOR_H_
     18 #define ART_COMPILER_OPTIMIZING_NODES_VECTOR_H_
     19 
     20 // This #include should never be used by compilation, because this header file (nodes_vector.h)
     21 // is included in the header file nodes.h itself. However it gives editing tools better context.
     22 #include "nodes.h"
     23 
     24 namespace art {
     25 
     26 // Memory alignment, represented as an offset relative to a base, where 0 <= offset < base,
     27 // and base is a power of two. For example, the value Alignment(16, 0) means memory is
     28 // perfectly aligned at a 16-byte boundary, whereas the value Alignment(16, 4) means
     29 // memory is always exactly 4 bytes above such a boundary.
     30 class Alignment {
     31  public:
     32   Alignment(size_t base, size_t offset) : base_(base), offset_(offset) {
     33     DCHECK_LT(offset, base);
     34     DCHECK(IsPowerOfTwo(base));
     35   }
     36 
     37   // Returns true if memory is at least aligned at the given boundary.
     38   // Assumes requested base is power of two.
     39   bool IsAlignedAt(size_t base) const {
     40     DCHECK_NE(0u, base);
     41     DCHECK(IsPowerOfTwo(base));
     42     return ((offset_ | base_) & (base - 1u)) == 0;
     43   }
     44 
     45   size_t Base() const { return base_; }
     46 
     47   size_t Offset() const { return offset_; }
     48 
     49   std::string ToString() const {
     50     return "ALIGN(" + std::to_string(base_) + "," + std::to_string(offset_) + ")";
     51   }
     52 
     53   bool operator==(const Alignment& other) const {
     54     return base_ == other.base_ && offset_ == other.offset_;
     55   }
     56 
     57  private:
     58   size_t base_;
     59   size_t offset_;
     60 };
     61 
     62 //
     63 // Definitions of abstract vector operations in HIR.
     64 //
     65 
     66 // Abstraction of a vector operation, i.e., an operation that performs
     67 // GetVectorLength() x GetPackedType() operations simultaneously.
     68 class HVecOperation : public HVariableInputSizeInstruction {
     69  public:
     70   // A SIMD operation looks like a FPU location.
     71   // TODO: we could introduce SIMD types in HIR.
     72   static constexpr DataType::Type kSIMDType = DataType::Type::kFloat64;
     73 
     74   HVecOperation(InstructionKind kind,
     75                 ArenaAllocator* allocator,
     76                 DataType::Type packed_type,
     77                 SideEffects side_effects,
     78                 size_t number_of_inputs,
     79                 size_t vector_length,
     80                 uint32_t dex_pc)
     81       : HVariableInputSizeInstruction(kind,
     82                                       side_effects,
     83                                       dex_pc,
     84                                       allocator,
     85                                       number_of_inputs,
     86                                       kArenaAllocVectorNode),
     87         vector_length_(vector_length) {
     88     SetPackedField<TypeField>(packed_type);
     89     DCHECK_LT(1u, vector_length);
     90   }
     91 
     92   // Returns the number of elements packed in a vector.
     93   size_t GetVectorLength() const {
     94     return vector_length_;
     95   }
     96 
     97   // Returns the number of bytes in a full vector.
     98   size_t GetVectorNumberOfBytes() const {
     99     return vector_length_ * DataType::Size(GetPackedType());
    100   }
    101 
    102   // Returns the type of the vector operation.
    103   DataType::Type GetType() const OVERRIDE {
    104     return kSIMDType;
    105   }
    106 
    107   // Returns the true component type packed in a vector.
    108   DataType::Type GetPackedType() const {
    109     return GetPackedField<TypeField>();
    110   }
    111 
    112   // Assumes vector nodes cannot be moved by default. Each concrete implementation
    113   // that can be moved should override this method and return true.
    114   //
    115   // Note: similar approach is used for instruction scheduling (if it is turned on for the target):
    116   // by default HScheduler::IsSchedulable returns false for a particular HVecOperation.
    117   // HScheduler${ARCH}::IsSchedulable can be overridden to return true for an instruction (see
    118   // scheduler_arm64.h for example) if it is safe to schedule it; in this case one *must* also
    119   // look at/update HScheduler${ARCH}::IsSchedulingBarrier for this instruction.
    120   //
    121   // Note: For newly introduced vector instructions HScheduler${ARCH}::IsSchedulingBarrier must be
    122   // altered to return true if the instruction might reside outside the SIMD loop body since SIMD
    123   // registers are not kept alive across vector loop boundaries (yet).
    124   bool CanBeMoved() const OVERRIDE { return false; }
    125 
    126   // Tests if all data of a vector node (vector length and packed type) is equal.
    127   // Each concrete implementation that adds more fields should test equality of
    128   // those fields in its own method *and* call all super methods.
    129   bool InstructionDataEquals(const HInstruction* other) const OVERRIDE {
    130     DCHECK(other->IsVecOperation());
    131     const HVecOperation* o = other->AsVecOperation();
    132     return GetVectorLength() == o->GetVectorLength() && GetPackedType() == o->GetPackedType();
    133   }
    134 
    135   // Maps an integral type to the same-size signed type and leaves other types alone.
    136   static DataType::Type ToSignedType(DataType::Type type) {
    137     switch (type) {
    138       case DataType::Type::kBool:  // 1-byte storage unit
    139       case DataType::Type::kUint8:
    140         return DataType::Type::kInt8;
    141       case DataType::Type::kUint16:
    142         return DataType::Type::kInt16;
    143       default:
    144         DCHECK(type != DataType::Type::kVoid && type != DataType::Type::kReference) << type;
    145         return type;
    146     }
    147   }
    148 
    149   // Maps an integral type to the same-size unsigned type and leaves other types alone.
    150   static DataType::Type ToUnsignedType(DataType::Type type) {
    151     switch (type) {
    152       case DataType::Type::kBool:  // 1-byte storage unit
    153       case DataType::Type::kInt8:
    154         return DataType::Type::kUint8;
    155       case DataType::Type::kInt16:
    156         return DataType::Type::kUint16;
    157       default:
    158         DCHECK(type != DataType::Type::kVoid && type != DataType::Type::kReference) << type;
    159         return type;
    160     }
    161   }
    162 
    163   // Maps an integral type to the same-size (un)signed type. Leaves other types alone.
    164   static DataType::Type ToProperType(DataType::Type type, bool is_unsigned) {
    165     return is_unsigned ? ToUnsignedType(type) : ToSignedType(type);
    166   }
    167 
    168   // Helper method to determine if an instruction returns a SIMD value.
    169   // TODO: This method is needed until we introduce SIMD as proper type.
    170   static bool ReturnsSIMDValue(HInstruction* instruction) {
    171     if (instruction->IsVecOperation()) {
    172       return !instruction->IsVecExtractScalar();  // only scalar returning vec op
    173     } else if (instruction->IsPhi()) {
    174       // Vectorizer only uses Phis in reductions, so checking for a 2-way phi
    175       // with a direct vector operand as second argument suffices.
    176       return
    177           instruction->GetType() == kSIMDType &&
    178           instruction->InputCount() == 2 &&
    179           instruction->InputAt(1)->IsVecOperation();
    180     }
    181     return false;
    182   }
    183 
    184   DECLARE_ABSTRACT_INSTRUCTION(VecOperation);
    185 
    186  protected:
    187   // Additional packed bits.
    188   static constexpr size_t kFieldType = HInstruction::kNumberOfGenericPackedBits;
    189   static constexpr size_t kFieldTypeSize =
    190       MinimumBitsToStore(static_cast<size_t>(DataType::Type::kLast));
    191   static constexpr size_t kNumberOfVectorOpPackedBits = kFieldType + kFieldTypeSize;
    192   static_assert(kNumberOfVectorOpPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields.");
    193   using TypeField = BitField<DataType::Type, kFieldType, kFieldTypeSize>;
    194 
    195   DEFAULT_COPY_CONSTRUCTOR(VecOperation);
    196 
    197  private:
    198   const size_t vector_length_;
    199 };
    200 
    201 // Abstraction of a unary vector operation.
    202 class HVecUnaryOperation : public HVecOperation {
    203  public:
    204   HVecUnaryOperation(InstructionKind kind,
    205                      ArenaAllocator* allocator,
    206                      HInstruction* input,
    207                      DataType::Type packed_type,
    208                      size_t vector_length,
    209                      uint32_t dex_pc)
    210       : HVecOperation(kind,
    211                       allocator,
    212                       packed_type,
    213                       SideEffects::None(),
    214                       /* number_of_inputs */ 1,
    215                       vector_length,
    216                       dex_pc) {
    217     SetRawInputAt(0, input);
    218   }
    219 
    220   HInstruction* GetInput() const { return InputAt(0); }
    221 
    222   DECLARE_ABSTRACT_INSTRUCTION(VecUnaryOperation);
    223 
    224  protected:
    225   DEFAULT_COPY_CONSTRUCTOR(VecUnaryOperation);
    226 };
    227 
    228 // Abstraction of a binary vector operation.
    229 class HVecBinaryOperation : public HVecOperation {
    230  public:
    231   HVecBinaryOperation(InstructionKind kind,
    232                       ArenaAllocator* allocator,
    233                       HInstruction* left,
    234                       HInstruction* right,
    235                       DataType::Type packed_type,
    236                       size_t vector_length,
    237                       uint32_t dex_pc)
    238       : HVecOperation(kind,
    239                       allocator,
    240                       packed_type,
    241                       SideEffects::None(),
    242                       /* number_of_inputs */ 2,
    243                       vector_length,
    244                       dex_pc) {
    245     SetRawInputAt(0, left);
    246     SetRawInputAt(1, right);
    247   }
    248 
    249   HInstruction* GetLeft() const { return InputAt(0); }
    250   HInstruction* GetRight() const { return InputAt(1); }
    251 
    252   DECLARE_ABSTRACT_INSTRUCTION(VecBinaryOperation);
    253 
    254  protected:
    255   DEFAULT_COPY_CONSTRUCTOR(VecBinaryOperation);
    256 };
    257 
    258 // Abstraction of a vector operation that references memory, with an alignment.
    259 // The Android runtime guarantees elements have at least natural alignment.
    260 class HVecMemoryOperation : public HVecOperation {
    261  public:
    262   HVecMemoryOperation(InstructionKind kind,
    263                       ArenaAllocator* allocator,
    264                       DataType::Type packed_type,
    265                       SideEffects side_effects,
    266                       size_t number_of_inputs,
    267                       size_t vector_length,
    268                       uint32_t dex_pc)
    269       : HVecOperation(kind,
    270                       allocator,
    271                       packed_type,
    272                       side_effects,
    273                       number_of_inputs,
    274                       vector_length,
    275                       dex_pc),
    276         alignment_(DataType::Size(packed_type), 0) {
    277     DCHECK_GE(number_of_inputs, 2u);
    278   }
    279 
    280   void SetAlignment(Alignment alignment) { alignment_ = alignment; }
    281 
    282   Alignment GetAlignment() const { return alignment_; }
    283 
    284   HInstruction* GetArray() const { return InputAt(0); }
    285   HInstruction* GetIndex() const { return InputAt(1); }
    286 
    287   bool InstructionDataEquals(const HInstruction* other) const OVERRIDE {
    288     DCHECK(other->IsVecMemoryOperation());
    289     const HVecMemoryOperation* o = other->AsVecMemoryOperation();
    290     return HVecOperation::InstructionDataEquals(o) && GetAlignment() == o->GetAlignment();
    291   }
    292 
    293   DECLARE_ABSTRACT_INSTRUCTION(VecMemoryOperation);
    294 
    295  protected:
    296   DEFAULT_COPY_CONSTRUCTOR(VecMemoryOperation);
    297 
    298  private:
    299   Alignment alignment_;
    300 };
    301 
    302 // Packed type consistency checker ("same vector length" integral types may mix freely).
    303 // Tests relaxed type consistency in which packed same-size integral types can co-exist,
    304 // but other type mixes are an error.
    305 inline static bool HasConsistentPackedTypes(HInstruction* input, DataType::Type type) {
    306   if (input->IsPhi()) {
    307     return input->GetType() == HVecOperation::kSIMDType;  // carries SIMD
    308   }
    309   DCHECK(input->IsVecOperation());
    310   DataType::Type input_type = input->AsVecOperation()->GetPackedType();
    311   DCHECK_EQ(HVecOperation::ToUnsignedType(input_type) == HVecOperation::ToUnsignedType(type),
    312             HVecOperation::ToSignedType(input_type) == HVecOperation::ToSignedType(type));
    313   return HVecOperation::ToSignedType(input_type) == HVecOperation::ToSignedType(type);
    314 }
    315 
    316 //
    317 // Definitions of concrete unary vector operations in HIR.
    318 //
    319 
    320 // Replicates the given scalar into a vector,
    321 // viz. replicate(x) = [ x, .. , x ].
    322 class HVecReplicateScalar FINAL : public HVecUnaryOperation {
    323  public:
    324   HVecReplicateScalar(ArenaAllocator* allocator,
    325                       HInstruction* scalar,
    326                       DataType::Type packed_type,
    327                       size_t vector_length,
    328                       uint32_t dex_pc)
    329       : HVecUnaryOperation(
    330             kVecReplicateScalar, allocator, scalar, packed_type, vector_length, dex_pc) {
    331     DCHECK(!ReturnsSIMDValue(scalar));
    332   }
    333 
    334   // A replicate needs to stay in place, since SIMD registers are not
    335   // kept alive across vector loop boundaries (yet).
    336   bool CanBeMoved() const OVERRIDE { return false; }
    337 
    338   DECLARE_INSTRUCTION(VecReplicateScalar);
    339 
    340  protected:
    341   DEFAULT_COPY_CONSTRUCTOR(VecReplicateScalar);
    342 };
    343 
    344 // Extracts a particular scalar from the given vector,
    345 // viz. extract[ x1, .. , xn ] = x_i.
    346 //
    347 // TODO: for now only i == 1 case supported.
    348 class HVecExtractScalar FINAL : public HVecUnaryOperation {
    349  public:
    350   HVecExtractScalar(ArenaAllocator* allocator,
    351                     HInstruction* input,
    352                     DataType::Type packed_type,
    353                     size_t vector_length,
    354                     size_t index,
    355                     uint32_t dex_pc)
    356       : HVecUnaryOperation(
    357             kVecExtractScalar, allocator, input, packed_type, vector_length, dex_pc) {
    358     DCHECK(HasConsistentPackedTypes(input, packed_type));
    359     DCHECK_LT(index, vector_length);
    360     DCHECK_EQ(index, 0u);
    361   }
    362 
    363   // Yields a single component in the vector.
    364   DataType::Type GetType() const OVERRIDE {
    365     return GetPackedType();
    366   }
    367 
    368   // An extract needs to stay in place, since SIMD registers are not
    369   // kept alive across vector loop boundaries (yet).
    370   bool CanBeMoved() const OVERRIDE { return false; }
    371 
    372   DECLARE_INSTRUCTION(VecExtractScalar);
    373 
    374  protected:
    375   DEFAULT_COPY_CONSTRUCTOR(VecExtractScalar);
    376 };
    377 
    378 // Reduces the given vector into the first element as sum/min/max,
    379 // viz. sum-reduce[ x1, .. , xn ] = [ y, ---- ], where y = sum xi
    380 // and the "-" denotes "don't care" (implementation dependent).
    381 class HVecReduce FINAL : public HVecUnaryOperation {
    382  public:
    383   enum ReductionKind {
    384     kSum = 1,
    385     kMin = 2,
    386     kMax = 3
    387   };
    388 
    389   HVecReduce(ArenaAllocator* allocator,
    390              HInstruction* input,
    391              DataType::Type packed_type,
    392              size_t vector_length,
    393              ReductionKind kind,
    394              uint32_t dex_pc)
    395       : HVecUnaryOperation(kVecReduce, allocator, input, packed_type, vector_length, dex_pc),
    396         kind_(kind) {
    397     DCHECK(HasConsistentPackedTypes(input, packed_type));
    398   }
    399 
    400   ReductionKind GetKind() const { return kind_; }
    401 
    402   bool CanBeMoved() const OVERRIDE { return true; }
    403 
    404   bool InstructionDataEquals(const HInstruction* other) const OVERRIDE {
    405     DCHECK(other->IsVecReduce());
    406     const HVecReduce* o = other->AsVecReduce();
    407     return HVecOperation::InstructionDataEquals(o) && GetKind() == o->GetKind();
    408   }
    409 
    410   DECLARE_INSTRUCTION(VecReduce);
    411 
    412  protected:
    413   DEFAULT_COPY_CONSTRUCTOR(VecReduce);
    414 
    415  private:
    416   const ReductionKind kind_;
    417 };
    418 
    419 // Converts every component in the vector,
    420 // viz. cnv[ x1, .. , xn ]  = [ cnv(x1), .. , cnv(xn) ].
    421 class HVecCnv FINAL : public HVecUnaryOperation {
    422  public:
    423   HVecCnv(ArenaAllocator* allocator,
    424           HInstruction* input,
    425           DataType::Type packed_type,
    426           size_t vector_length,
    427           uint32_t dex_pc)
    428       : HVecUnaryOperation(kVecCnv, allocator, input, packed_type, vector_length, dex_pc) {
    429     DCHECK(input->IsVecOperation());
    430     DCHECK_NE(GetInputType(), GetResultType());  // actual convert
    431   }
    432 
    433   DataType::Type GetInputType() const { return InputAt(0)->AsVecOperation()->GetPackedType(); }
    434   DataType::Type GetResultType() const { return GetPackedType(); }
    435 
    436   bool CanBeMoved() const OVERRIDE { return true; }
    437 
    438   DECLARE_INSTRUCTION(VecCnv);
    439 
    440  protected:
    441   DEFAULT_COPY_CONSTRUCTOR(VecCnv);
    442 };
    443 
    444 // Negates every component in the vector,
    445 // viz. neg[ x1, .. , xn ]  = [ -x1, .. , -xn ].
    446 class HVecNeg FINAL : public HVecUnaryOperation {
    447  public:
    448   HVecNeg(ArenaAllocator* allocator,
    449           HInstruction* input,
    450           DataType::Type packed_type,
    451           size_t vector_length,
    452           uint32_t dex_pc)
    453       : HVecUnaryOperation(kVecNeg, allocator, input, packed_type, vector_length, dex_pc) {
    454     DCHECK(HasConsistentPackedTypes(input, packed_type));
    455   }
    456 
    457   bool CanBeMoved() const OVERRIDE { return true; }
    458 
    459   DECLARE_INSTRUCTION(VecNeg);
    460 
    461  protected:
    462   DEFAULT_COPY_CONSTRUCTOR(VecNeg);
    463 };
    464 
    465 // Takes absolute value of every component in the vector,
    466 // viz. abs[ x1, .. , xn ]  = [ |x1|, .. , |xn| ]
    467 // for signed operand x.
    468 class HVecAbs FINAL : public HVecUnaryOperation {
    469  public:
    470   HVecAbs(ArenaAllocator* allocator,
    471           HInstruction* input,
    472           DataType::Type packed_type,
    473           size_t vector_length,
    474           uint32_t dex_pc)
    475       : HVecUnaryOperation(kVecAbs, allocator, input, packed_type, vector_length, dex_pc) {
    476     DCHECK(HasConsistentPackedTypes(input, packed_type));
    477   }
    478 
    479   bool CanBeMoved() const OVERRIDE { return true; }
    480 
    481   DECLARE_INSTRUCTION(VecAbs);
    482 
    483  protected:
    484   DEFAULT_COPY_CONSTRUCTOR(VecAbs);
    485 };
    486 
    487 // Bitwise- or boolean-nots every component in the vector,
    488 // viz. not[ x1, .. , xn ]  = [ ~x1, .. , ~xn ], or
    489 //      not[ x1, .. , xn ]  = [ !x1, .. , !xn ] for boolean.
    490 class HVecNot FINAL : public HVecUnaryOperation {
    491  public:
    492   HVecNot(ArenaAllocator* allocator,
    493           HInstruction* input,
    494           DataType::Type packed_type,
    495           size_t vector_length,
    496           uint32_t dex_pc)
    497       : HVecUnaryOperation(kVecNot, allocator, input, packed_type, vector_length, dex_pc) {
    498     DCHECK(input->IsVecOperation());
    499   }
    500 
    501   bool CanBeMoved() const OVERRIDE { return true; }
    502 
    503   DECLARE_INSTRUCTION(VecNot);
    504 
    505  protected:
    506   DEFAULT_COPY_CONSTRUCTOR(VecNot);
    507 };
    508 
    509 //
    510 // Definitions of concrete binary vector operations in HIR.
    511 //
    512 
    513 // Adds every component in the two vectors,
    514 // viz. [ x1, .. , xn ] + [ y1, .. , yn ] = [ x1 + y1, .. , xn + yn ].
    515 class HVecAdd FINAL : public HVecBinaryOperation {
    516  public:
    517   HVecAdd(ArenaAllocator* allocator,
    518           HInstruction* left,
    519           HInstruction* right,
    520           DataType::Type packed_type,
    521           size_t vector_length,
    522           uint32_t dex_pc)
    523       : HVecBinaryOperation(kVecAdd, allocator, left, right, packed_type, vector_length, dex_pc) {
    524     DCHECK(HasConsistentPackedTypes(left, packed_type));
    525     DCHECK(HasConsistentPackedTypes(right, packed_type));
    526   }
    527 
    528   bool CanBeMoved() const OVERRIDE { return true; }
    529 
    530   DECLARE_INSTRUCTION(VecAdd);
    531 
    532  protected:
    533   DEFAULT_COPY_CONSTRUCTOR(VecAdd);
    534 };
    535 
    536 // Performs halving add on every component in the two vectors, viz.
    537 // rounded   [ x1, .. , xn ] hradd [ y1, .. , yn ] = [ (x1 + y1 + 1) >> 1, .. , (xn + yn + 1) >> 1 ]
    538 // truncated [ x1, .. , xn ] hadd  [ y1, .. , yn ] = [ (x1 + y1)     >> 1, .. , (xn + yn )    >> 1 ]
    539 // for either both signed or both unsigned operands x, y (reflected in packed_type).
    540 class HVecHalvingAdd FINAL : public HVecBinaryOperation {
    541  public:
    542   HVecHalvingAdd(ArenaAllocator* allocator,
    543                  HInstruction* left,
    544                  HInstruction* right,
    545                  DataType::Type packed_type,
    546                  size_t vector_length,
    547                  bool is_rounded,
    548                  uint32_t dex_pc)
    549       : HVecBinaryOperation(
    550             kVecHalvingAdd, allocator, left, right, packed_type, vector_length, dex_pc) {
    551     DCHECK(HasConsistentPackedTypes(left, packed_type));
    552     DCHECK(HasConsistentPackedTypes(right, packed_type));
    553     SetPackedFlag<kFieldHAddIsRounded>(is_rounded);
    554   }
    555 
    556   bool IsRounded() const { return GetPackedFlag<kFieldHAddIsRounded>(); }
    557 
    558   bool CanBeMoved() const OVERRIDE { return true; }
    559 
    560   bool InstructionDataEquals(const HInstruction* other) const OVERRIDE {
    561     DCHECK(other->IsVecHalvingAdd());
    562     const HVecHalvingAdd* o = other->AsVecHalvingAdd();
    563     return HVecOperation::InstructionDataEquals(o) && IsRounded() == o->IsRounded();
    564   }
    565 
    566   DECLARE_INSTRUCTION(VecHalvingAdd);
    567 
    568  protected:
    569   DEFAULT_COPY_CONSTRUCTOR(VecHalvingAdd);
    570 
    571  private:
    572   // Additional packed bits.
    573   static constexpr size_t kFieldHAddIsRounded = HVecOperation::kNumberOfVectorOpPackedBits;
    574   static constexpr size_t kNumberOfHAddPackedBits = kFieldHAddIsRounded + 1;
    575   static_assert(kNumberOfHAddPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields.");
    576 };
    577 
    578 // Subtracts every component in the two vectors,
    579 // viz. [ x1, .. , xn ] - [ y1, .. , yn ] = [ x1 - y1, .. , xn - yn ].
    580 class HVecSub FINAL : public HVecBinaryOperation {
    581  public:
    582   HVecSub(ArenaAllocator* allocator,
    583           HInstruction* left,
    584           HInstruction* right,
    585           DataType::Type packed_type,
    586           size_t vector_length,
    587           uint32_t dex_pc)
    588       : HVecBinaryOperation(kVecSub, allocator, left, right, packed_type, vector_length, dex_pc) {
    589     DCHECK(HasConsistentPackedTypes(left, packed_type));
    590     DCHECK(HasConsistentPackedTypes(right, packed_type));
    591   }
    592 
    593   bool CanBeMoved() const OVERRIDE { return true; }
    594 
    595   DECLARE_INSTRUCTION(VecSub);
    596 
    597  protected:
    598   DEFAULT_COPY_CONSTRUCTOR(VecSub);
    599 };
    600 
    601 // Multiplies every component in the two vectors,
    602 // viz. [ x1, .. , xn ] * [ y1, .. , yn ] = [ x1 * y1, .. , xn * yn ].
    603 class HVecMul FINAL : public HVecBinaryOperation {
    604  public:
    605   HVecMul(ArenaAllocator* allocator,
    606           HInstruction* left,
    607           HInstruction* right,
    608           DataType::Type packed_type,
    609           size_t vector_length,
    610           uint32_t dex_pc)
    611       : HVecBinaryOperation(kVecMul, allocator, left, right, packed_type, vector_length, dex_pc) {
    612     DCHECK(HasConsistentPackedTypes(left, packed_type));
    613     DCHECK(HasConsistentPackedTypes(right, packed_type));
    614   }
    615 
    616   bool CanBeMoved() const OVERRIDE { return true; }
    617 
    618   DECLARE_INSTRUCTION(VecMul);
    619 
    620  protected:
    621   DEFAULT_COPY_CONSTRUCTOR(VecMul);
    622 };
    623 
    624 // Divides every component in the two vectors,
    625 // viz. [ x1, .. , xn ] / [ y1, .. , yn ] = [ x1 / y1, .. , xn / yn ].
    626 class HVecDiv FINAL : public HVecBinaryOperation {
    627  public:
    628   HVecDiv(ArenaAllocator* allocator,
    629           HInstruction* left,
    630           HInstruction* right,
    631           DataType::Type packed_type,
    632           size_t vector_length,
    633           uint32_t dex_pc)
    634       : HVecBinaryOperation(kVecDiv, allocator, left, right, packed_type, vector_length, dex_pc) {
    635     DCHECK(HasConsistentPackedTypes(left, packed_type));
    636     DCHECK(HasConsistentPackedTypes(right, packed_type));
    637   }
    638 
    639   bool CanBeMoved() const OVERRIDE { return true; }
    640 
    641   DECLARE_INSTRUCTION(VecDiv);
    642 
    643  protected:
    644   DEFAULT_COPY_CONSTRUCTOR(VecDiv);
    645 };
    646 
    647 // Takes minimum of every component in the two vectors,
    648 // viz. MIN( [ x1, .. , xn ] , [ y1, .. , yn ]) = [ min(x1, y1), .. , min(xn, yn) ]
    649 // for either both signed or both unsigned operands x, y (reflected in packed_type).
    650 class HVecMin FINAL : public HVecBinaryOperation {
    651  public:
    652   HVecMin(ArenaAllocator* allocator,
    653           HInstruction* left,
    654           HInstruction* right,
    655           DataType::Type packed_type,
    656           size_t vector_length,
    657           uint32_t dex_pc)
    658       : HVecBinaryOperation(kVecMin, allocator, left, right, packed_type, vector_length, dex_pc) {
    659     DCHECK(HasConsistentPackedTypes(left, packed_type));
    660     DCHECK(HasConsistentPackedTypes(right, packed_type));
    661   }
    662 
    663   bool CanBeMoved() const OVERRIDE { return true; }
    664 
    665   DECLARE_INSTRUCTION(VecMin);
    666 
    667  protected:
    668   DEFAULT_COPY_CONSTRUCTOR(VecMin);
    669 };
    670 
    671 // Takes maximum of every component in the two vectors,
    672 // viz. MAX( [ x1, .. , xn ] , [ y1, .. , yn ]) = [ max(x1, y1), .. , max(xn, yn) ]
    673 // for either both signed or both unsigned operands x, y (reflected in packed_type).
    674 class HVecMax FINAL : public HVecBinaryOperation {
    675  public:
    676   HVecMax(ArenaAllocator* allocator,
    677           HInstruction* left,
    678           HInstruction* right,
    679           DataType::Type packed_type,
    680           size_t vector_length,
    681           uint32_t dex_pc)
    682       : HVecBinaryOperation(kVecMax, allocator, left, right, packed_type, vector_length, dex_pc) {
    683     DCHECK(HasConsistentPackedTypes(left, packed_type));
    684     DCHECK(HasConsistentPackedTypes(right, packed_type));
    685   }
    686 
    687   bool CanBeMoved() const OVERRIDE { return true; }
    688 
    689   DECLARE_INSTRUCTION(VecMax);
    690 
    691  protected:
    692   DEFAULT_COPY_CONSTRUCTOR(VecMax);
    693 };
    694 
    695 // Bitwise-ands every component in the two vectors,
    696 // viz. [ x1, .. , xn ] & [ y1, .. , yn ] = [ x1 & y1, .. , xn & yn ].
    697 class HVecAnd FINAL : public HVecBinaryOperation {
    698  public:
    699   HVecAnd(ArenaAllocator* allocator,
    700           HInstruction* left,
    701           HInstruction* right,
    702           DataType::Type packed_type,
    703           size_t vector_length,
    704           uint32_t dex_pc)
    705       : HVecBinaryOperation(kVecAnd, allocator, left, right, packed_type, vector_length, dex_pc) {
    706     DCHECK(left->IsVecOperation() && right->IsVecOperation());
    707   }
    708 
    709   bool CanBeMoved() const OVERRIDE { return true; }
    710 
    711   DECLARE_INSTRUCTION(VecAnd);
    712 
    713  protected:
    714   DEFAULT_COPY_CONSTRUCTOR(VecAnd);
    715 };
    716 
    717 // Bitwise-and-nots every component in the two vectors,
    718 // viz. [ x1, .. , xn ] and-not [ y1, .. , yn ] = [ ~x1 & y1, .. , ~xn & yn ].
    719 class HVecAndNot FINAL : public HVecBinaryOperation {
    720  public:
    721   HVecAndNot(ArenaAllocator* allocator,
    722              HInstruction* left,
    723              HInstruction* right,
    724              DataType::Type packed_type,
    725              size_t vector_length,
    726              uint32_t dex_pc)
    727          : HVecBinaryOperation(
    728                kVecAndNot, allocator, left, right, packed_type, vector_length, dex_pc) {
    729     DCHECK(left->IsVecOperation() && right->IsVecOperation());
    730   }
    731 
    732   bool CanBeMoved() const OVERRIDE { return true; }
    733 
    734   DECLARE_INSTRUCTION(VecAndNot);
    735 
    736  protected:
    737   DEFAULT_COPY_CONSTRUCTOR(VecAndNot);
    738 };
    739 
    740 // Bitwise-ors every component in the two vectors,
    741 // viz. [ x1, .. , xn ] | [ y1, .. , yn ] = [ x1 | y1, .. , xn | yn ].
    742 class HVecOr FINAL : public HVecBinaryOperation {
    743  public:
    744   HVecOr(ArenaAllocator* allocator,
    745          HInstruction* left,
    746          HInstruction* right,
    747          DataType::Type packed_type,
    748          size_t vector_length,
    749          uint32_t dex_pc)
    750       : HVecBinaryOperation(kVecOr, allocator, left, right, packed_type, vector_length, dex_pc) {
    751     DCHECK(left->IsVecOperation() && right->IsVecOperation());
    752   }
    753 
    754   bool CanBeMoved() const OVERRIDE { return true; }
    755 
    756   DECLARE_INSTRUCTION(VecOr);
    757 
    758  protected:
    759   DEFAULT_COPY_CONSTRUCTOR(VecOr);
    760 };
    761 
    762 // Bitwise-xors every component in the two vectors,
    763 // viz. [ x1, .. , xn ] ^ [ y1, .. , yn ] = [ x1 ^ y1, .. , xn ^ yn ].
    764 class HVecXor FINAL : public HVecBinaryOperation {
    765  public:
    766   HVecXor(ArenaAllocator* allocator,
    767           HInstruction* left,
    768           HInstruction* right,
    769           DataType::Type packed_type,
    770           size_t vector_length,
    771           uint32_t dex_pc)
    772       : HVecBinaryOperation(kVecXor, allocator, left, right, packed_type, vector_length, dex_pc) {
    773     DCHECK(left->IsVecOperation() && right->IsVecOperation());
    774   }
    775 
    776   bool CanBeMoved() const OVERRIDE { return true; }
    777 
    778   DECLARE_INSTRUCTION(VecXor);
    779 
    780  protected:
    781   DEFAULT_COPY_CONSTRUCTOR(VecXor);
    782 };
    783 
    784 // Logically shifts every component in the vector left by the given distance,
    785 // viz. [ x1, .. , xn ] << d = [ x1 << d, .. , xn << d ].
    786 class HVecShl FINAL : public HVecBinaryOperation {
    787  public:
    788   HVecShl(ArenaAllocator* allocator,
    789           HInstruction* left,
    790           HInstruction* right,
    791           DataType::Type packed_type,
    792           size_t vector_length,
    793           uint32_t dex_pc)
    794       : HVecBinaryOperation(kVecShl, allocator, left, right, packed_type, vector_length, dex_pc) {
    795     DCHECK(HasConsistentPackedTypes(left, packed_type));
    796   }
    797 
    798   bool CanBeMoved() const OVERRIDE { return true; }
    799 
    800   DECLARE_INSTRUCTION(VecShl);
    801 
    802  protected:
    803   DEFAULT_COPY_CONSTRUCTOR(VecShl);
    804 };
    805 
    806 // Arithmetically shifts every component in the vector right by the given distance,
    807 // viz. [ x1, .. , xn ] >> d = [ x1 >> d, .. , xn >> d ].
    808 class HVecShr FINAL : public HVecBinaryOperation {
    809  public:
    810   HVecShr(ArenaAllocator* allocator,
    811           HInstruction* left,
    812           HInstruction* right,
    813           DataType::Type packed_type,
    814           size_t vector_length,
    815           uint32_t dex_pc)
    816       : HVecBinaryOperation(kVecShr, allocator, left, right, packed_type, vector_length, dex_pc) {
    817     DCHECK(HasConsistentPackedTypes(left, packed_type));
    818   }
    819 
    820   bool CanBeMoved() const OVERRIDE { return true; }
    821 
    822   DECLARE_INSTRUCTION(VecShr);
    823 
    824  protected:
    825   DEFAULT_COPY_CONSTRUCTOR(VecShr);
    826 };
    827 
    828 // Logically shifts every component in the vector right by the given distance,
    829 // viz. [ x1, .. , xn ] >>> d = [ x1 >>> d, .. , xn >>> d ].
    830 class HVecUShr FINAL : public HVecBinaryOperation {
    831  public:
    832   HVecUShr(ArenaAllocator* allocator,
    833            HInstruction* left,
    834            HInstruction* right,
    835            DataType::Type packed_type,
    836            size_t vector_length,
    837            uint32_t dex_pc)
    838       : HVecBinaryOperation(kVecUShr, allocator, left, right, packed_type, vector_length, dex_pc) {
    839     DCHECK(HasConsistentPackedTypes(left, packed_type));
    840   }
    841 
    842   bool CanBeMoved() const OVERRIDE { return true; }
    843 
    844   DECLARE_INSTRUCTION(VecUShr);
    845 
    846  protected:
    847   DEFAULT_COPY_CONSTRUCTOR(VecUShr);
    848 };
    849 
    850 //
    851 // Definitions of concrete miscellaneous vector operations in HIR.
    852 //
    853 
    854 // Assigns the given scalar elements to a vector,
    855 // viz. set( array(x1, .. , xn) ) = [ x1, .. ,            xn ] if n == m,
    856 //      set( array(x1, .. , xm) ) = [ x1, .. , xm, 0, .. , 0 ] if m <  n.
    857 class HVecSetScalars FINAL : public HVecOperation {
    858  public:
    859   HVecSetScalars(ArenaAllocator* allocator,
    860                  HInstruction* scalars[],
    861                  DataType::Type packed_type,
    862                  size_t vector_length,
    863                  size_t number_of_scalars,
    864                  uint32_t dex_pc)
    865       : HVecOperation(kVecSetScalars,
    866                       allocator,
    867                       packed_type,
    868                       SideEffects::None(),
    869                       number_of_scalars,
    870                       vector_length,
    871                       dex_pc) {
    872     for (size_t i = 0; i < number_of_scalars; i++) {
    873       DCHECK(!ReturnsSIMDValue(scalars[i]));
    874       SetRawInputAt(0, scalars[i]);
    875     }
    876   }
    877 
    878   // Setting scalars needs to stay in place, since SIMD registers are not
    879   // kept alive across vector loop boundaries (yet).
    880   bool CanBeMoved() const OVERRIDE { return false; }
    881 
    882   DECLARE_INSTRUCTION(VecSetScalars);
    883 
    884  protected:
    885   DEFAULT_COPY_CONSTRUCTOR(VecSetScalars);
    886 };
    887 
    888 // Multiplies every component in the two vectors, adds the result vector to the accumulator vector,
    889 // viz. [ a1, .. , an ] + [ x1, .. , xn ] * [ y1, .. , yn ] = [ a1 + x1 * y1, .. , an + xn * yn ].
    890 class HVecMultiplyAccumulate FINAL : public HVecOperation {
    891  public:
    892   HVecMultiplyAccumulate(ArenaAllocator* allocator,
    893                          InstructionKind op,
    894                          HInstruction* accumulator,
    895                          HInstruction* mul_left,
    896                          HInstruction* mul_right,
    897                          DataType::Type packed_type,
    898                          size_t vector_length,
    899                          uint32_t dex_pc)
    900       : HVecOperation(kVecMultiplyAccumulate,
    901                       allocator,
    902                       packed_type,
    903                       SideEffects::None(),
    904                       /* number_of_inputs */ 3,
    905                       vector_length,
    906                       dex_pc),
    907         op_kind_(op) {
    908     DCHECK(op == InstructionKind::kAdd || op == InstructionKind::kSub);
    909     DCHECK(HasConsistentPackedTypes(accumulator, packed_type));
    910     DCHECK(HasConsistentPackedTypes(mul_left, packed_type));
    911     DCHECK(HasConsistentPackedTypes(mul_right, packed_type));
    912     SetRawInputAt(0, accumulator);
    913     SetRawInputAt(1, mul_left);
    914     SetRawInputAt(2, mul_right);
    915   }
    916 
    917   bool CanBeMoved() const OVERRIDE { return true; }
    918 
    919   bool InstructionDataEquals(const HInstruction* other) const OVERRIDE {
    920     DCHECK(other->IsVecMultiplyAccumulate());
    921     const HVecMultiplyAccumulate* o = other->AsVecMultiplyAccumulate();
    922     return HVecOperation::InstructionDataEquals(o) && GetOpKind() == o->GetOpKind();
    923   }
    924 
    925   InstructionKind GetOpKind() const { return op_kind_; }
    926 
    927   DECLARE_INSTRUCTION(VecMultiplyAccumulate);
    928 
    929  protected:
    930   DEFAULT_COPY_CONSTRUCTOR(VecMultiplyAccumulate);
    931 
    932  private:
    933   // Indicates if this is a MADD or MSUB.
    934   const InstructionKind op_kind_;
    935 };
    936 
    937 // Takes the absolute difference of two vectors, and adds the results to
    938 // same-precision or wider-precision components in the accumulator,
    939 // viz. SAD([ a1, .. , am ], [ x1, .. , xn ], [ y1, .. , yn ]) =
    940 //          [ a1 + sum abs(xi-yi), .. , am + sum abs(xj-yj) ],
    941 //      for m <= n, non-overlapping sums, and signed operands x, y.
    942 class HVecSADAccumulate FINAL : public HVecOperation {
    943  public:
    944   HVecSADAccumulate(ArenaAllocator* allocator,
    945                     HInstruction* accumulator,
    946                     HInstruction* sad_left,
    947                     HInstruction* sad_right,
    948                     DataType::Type packed_type,
    949                     size_t vector_length,
    950                     uint32_t dex_pc)
    951       : HVecOperation(kVecSADAccumulate,
    952                       allocator,
    953                       packed_type,
    954                       SideEffects::None(),
    955                       /* number_of_inputs */ 3,
    956                       vector_length,
    957                       dex_pc) {
    958     DCHECK(HasConsistentPackedTypes(accumulator, packed_type));
    959     DCHECK(sad_left->IsVecOperation());
    960     DCHECK(sad_right->IsVecOperation());
    961     DCHECK_EQ(ToSignedType(sad_left->AsVecOperation()->GetPackedType()),
    962               ToSignedType(sad_right->AsVecOperation()->GetPackedType()));
    963     SetRawInputAt(0, accumulator);
    964     SetRawInputAt(1, sad_left);
    965     SetRawInputAt(2, sad_right);
    966   }
    967 
    968   DECLARE_INSTRUCTION(VecSADAccumulate);
    969 
    970  protected:
    971   DEFAULT_COPY_CONSTRUCTOR(VecSADAccumulate);
    972 };
    973 
    974 // Loads a vector from memory, viz. load(mem, 1)
    975 // yield the vector [ mem(1), .. , mem(n) ].
    976 class HVecLoad FINAL : public HVecMemoryOperation {
    977  public:
    978   HVecLoad(ArenaAllocator* allocator,
    979            HInstruction* base,
    980            HInstruction* index,
    981            DataType::Type packed_type,
    982            SideEffects side_effects,
    983            size_t vector_length,
    984            bool is_string_char_at,
    985            uint32_t dex_pc)
    986       : HVecMemoryOperation(kVecLoad,
    987                             allocator,
    988                             packed_type,
    989                             side_effects,
    990                             /* number_of_inputs */ 2,
    991                             vector_length,
    992                             dex_pc) {
    993     SetRawInputAt(0, base);
    994     SetRawInputAt(1, index);
    995     SetPackedFlag<kFieldIsStringCharAt>(is_string_char_at);
    996   }
    997 
    998   bool IsStringCharAt() const { return GetPackedFlag<kFieldIsStringCharAt>(); }
    999 
   1000   bool CanBeMoved() const OVERRIDE { return true; }
   1001 
   1002   bool InstructionDataEquals(const HInstruction* other) const OVERRIDE {
   1003     DCHECK(other->IsVecLoad());
   1004     const HVecLoad* o = other->AsVecLoad();
   1005     return HVecMemoryOperation::InstructionDataEquals(o) && IsStringCharAt() == o->IsStringCharAt();
   1006   }
   1007 
   1008   DECLARE_INSTRUCTION(VecLoad);
   1009 
   1010  protected:
   1011   DEFAULT_COPY_CONSTRUCTOR(VecLoad);
   1012 
   1013  private:
   1014   // Additional packed bits.
   1015   static constexpr size_t kFieldIsStringCharAt = HVecOperation::kNumberOfVectorOpPackedBits;
   1016   static constexpr size_t kNumberOfVecLoadPackedBits = kFieldIsStringCharAt + 1;
   1017   static_assert(kNumberOfVecLoadPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields.");
   1018 };
   1019 
   1020 // Stores a vector to memory, viz. store(m, 1, [x1, .. , xn] )
   1021 // sets mem(1) = x1, .. , mem(n) = xn.
   1022 class HVecStore FINAL : public HVecMemoryOperation {
   1023  public:
   1024   HVecStore(ArenaAllocator* allocator,
   1025             HInstruction* base,
   1026             HInstruction* index,
   1027             HInstruction* value,
   1028             DataType::Type packed_type,
   1029             SideEffects side_effects,
   1030             size_t vector_length,
   1031             uint32_t dex_pc)
   1032       : HVecMemoryOperation(kVecStore,
   1033                             allocator,
   1034                             packed_type,
   1035                             side_effects,
   1036                             /* number_of_inputs */ 3,
   1037                             vector_length,
   1038                             dex_pc) {
   1039     DCHECK(HasConsistentPackedTypes(value, packed_type));
   1040     SetRawInputAt(0, base);
   1041     SetRawInputAt(1, index);
   1042     SetRawInputAt(2, value);
   1043   }
   1044 
   1045   // A store needs to stay in place.
   1046   bool CanBeMoved() const OVERRIDE { return false; }
   1047 
   1048   DECLARE_INSTRUCTION(VecStore);
   1049 
   1050  protected:
   1051   DEFAULT_COPY_CONSTRUCTOR(VecStore)
   1052 };
   1053 
   1054 }  // namespace art
   1055 
   1056 #endif  // ART_COMPILER_OPTIMIZING_NODES_VECTOR_H_
   1057