// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

#ifndef RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_ARM64_H_
#define RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_ARM64_H_

#if defined(DART_PRECOMPILED_RUNTIME)
#error "AOT runtime should not use compiler sources (including header files)"
#endif  // defined(DART_PRECOMPILED_RUNTIME)

#ifndef RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_H_
#error Do not include assembler_arm64.h directly; use assembler.h instead.
#endif

#include <functional>

#include "platform/assert.h"
#include "platform/utils.h"
#include "vm/class_id.h"
#include "vm/compiler/assembler/assembler_base.h"
#include "vm/constants.h"
#include "vm/hash_map.h"
#include "vm/simulator.h"

namespace dart {

// Forward declarations.
class FlowGraphCompiler;
class RuntimeEntry;
class RegisterSet;

namespace compiler {

class Immediate : public ValueObject {
 public:
  explicit Immediate(int64_t value) : value_(value) {}

  Immediate(const Immediate& other) : ValueObject(), value_(other.value_) {}
  Immediate& operator=(const Immediate& other) {
    value_ = other.value_;
    return *this;
  }

 private:
  int64_t value_;

  int64_t value() const { return value_; }

  friend class Assembler;
};

class Arm64Encode : public AllStatic {
 public:
  static inline uint32_t Rd(Register rd) {
    ASSERT(rd <= ZR);
    return static_cast<uint32_t>(ConcreteRegister(rd)) << kRdShift;
  }

  static inline uint32_t Rm(Register rm) {
    ASSERT(rm <= ZR);
    return static_cast<uint32_t>(ConcreteRegister(rm)) << kRmShift;
  }

  static inline uint32_t Rn(Register rn) {
    ASSERT(rn <= ZR);
    return static_cast<uint32_t>(ConcreteRegister(rn)) << kRnShift;
  }

  static inline uint32_t Ra(Register ra) {
    ASSERT(ra <= ZR);
    return static_cast<uint32_t>(ConcreteRegister(ra)) << kRaShift;
  }

  static inline uint32_t Rs(Register rs) {
    ASSERT(rs <= ZR);
    return static_cast<uint32_t>(ConcreteRegister(rs)) << kRsShift;
  }

  static inline uint32_t Rt(Register rt) {
    ASSERT(rt <= ZR);
    return static_cast<uint32_t>(ConcreteRegister(rt)) << kRtShift;
  }

  static inline uint32_t Rt2(Register rt2) {
    ASSERT(rt2 <= ZR);
    return static_cast<uint32_t>(ConcreteRegister(rt2)) << kRt2Shift;
  }
};

class Address : public ValueObject {
 public:
  Address(const Address& other)
      : ValueObject(),
        encoding_(other.encoding_),
        type_(other.type_),
        base_(other.base_) {}

  Address& operator=(const Address& other) {
    encoding_ = other.encoding_;
    type_ = other.type_;
    base_ = other.base_;
    return *this;
  }

  enum AddressType {
    Offset,
    PreIndex,
    PostIndex,
    PairOffset,
    PairPreIndex,
    PairPostIndex,
    Reg,
    PCOffset,
    Unknown,
  };

  // If we are doing pre-/post-indexing, and the base and result registers are
  // the same, then the result is unpredictable. This kind of instruction is
  // actually illegal on some microarchitectures.
  bool can_writeback_to(Register r) const {
    if (type() == PreIndex || type() == PostIndex || type() == PairPreIndex ||
        type() == PairPostIndex) {
      return base() != r;
    }
    return true;
  }

  // Offset is in bytes. For the unsigned imm12 case, we unscale based on the
  // operand size, and assert that offset is aligned accordingly.
  // For the smaller signed imm9 case, the offset is the number of bytes, but
  // is unscaled.
  Address(Register rn,
          int32_t offset = 0,
          AddressType at = Offset,
          OperandSize sz = kDoubleWord) {
    ASSERT((rn != kNoRegister) && (rn != R31) && (rn != ZR));
    ASSERT(CanHoldOffset(offset, at, sz));
    const int32_t scale = Log2OperandSizeBytes(sz);
    if ((at == Offset) && Utils::IsUint(12 + scale, offset) &&
        (offset == ((offset >> scale) << scale))) {
      encoding_ =
          B24 | ((offset >> scale) << kImm12Shift) | Arm64Encode::Rn(rn);
    } else if ((at == Offset) && Utils::IsInt(9, offset)) {
      encoding_ = ((offset & 0x1ff) << kImm9Shift) | Arm64Encode::Rn(rn);
    } else if ((at == PreIndex) || (at == PostIndex)) {
      ASSERT(Utils::IsInt(9, offset));
      int32_t idx = (at == PostIndex) ? B10 : (B11 | B10);
      encoding_ = idx | ((offset & 0x1ff) << kImm9Shift) | Arm64Encode::Rn(rn);
    } else {
      ASSERT((at == PairOffset) || (at == PairPreIndex) ||
             (at == PairPostIndex));
      ASSERT(Utils::IsInt(7 + scale, offset) &&
             (static_cast<uint32_t>(offset) ==
              ((static_cast<uint32_t>(offset) >> scale) << scale)));
      int32_t idx = 0;
      switch (at) {
        case PairPostIndex:
          idx = B23;
          break;
        case PairPreIndex:
          idx = B24 | B23;
          break;
        case PairOffset:
          idx = B24;
          break;
        default:
          UNREACHABLE();
          break;
      }
      encoding_ =
          idx |
          ((static_cast<uint32_t>(offset >> scale) << kImm7Shift) & kImm7Mask) |
          Arm64Encode::Rn(rn);
    }
    type_ = at;
    base_ = ConcreteRegister(rn);
  }

  // This addressing mode does not exist.
  Address(Register rn,
          Register offset,
          AddressType at,
          OperandSize sz = kDoubleWord);

  static bool CanHoldOffset(int32_t offset,
                            AddressType at = Offset,
                            OperandSize sz = kDoubleWord) {
    if (at == Offset) {
      // Offset fits in 12 bit unsigned and has right alignment for sz,
      // or fits in 9 bit signed offset with no alignment restriction.
      const int32_t scale = Log2OperandSizeBytes(sz);
      return (Utils::IsUint(12 + scale, offset) &&
              (offset == ((offset >> scale) << scale))) ||
             (Utils::IsInt(9, offset));
    } else if (at == PCOffset) {
      return Utils::IsInt(21, offset) && (offset == ((offset >> 2) << 2));
    } else if ((at == PreIndex) || (at == PostIndex)) {
      return Utils::IsInt(9, offset);
    } else {
      ASSERT((at == PairOffset) || (at == PairPreIndex) ||
             (at == PairPostIndex));
      const int32_t scale = Log2OperandSizeBytes(sz);
      return (Utils::IsInt(7 + scale, offset) &&
              (static_cast<uint32_t>(offset) ==
               ((static_cast<uint32_t>(offset) >> scale) << scale)));
    }
  }

  // PC-relative load address.
  static Address PC(int32_t pc_off) {
    ASSERT(CanHoldOffset(pc_off, PCOffset));
    Address addr;
    addr.encoding_ = (((pc_off >> 2) << kImm19Shift) & kImm19Mask);
    addr.base_ = kNoRegister;
    addr.type_ = PCOffset;
    return addr;
  }

  static Address Pair(Register rn,
                      int32_t offset = 0,
                      AddressType at = PairOffset,
                      OperandSize sz = kDoubleWord) {
    return Address(rn, offset, at, sz);
  }

  // This addressing mode does not exist.
  static Address PC(Register r);

  enum Scaling {
    Unscaled,
    Scaled,
  };

  // Base register rn with offset rm. rm is sign-extended according to ext.
  // If ext is UXTX, rm may be optionally scaled by the
  // Log2OperandSize (specified by the instruction).
  Address(Register rn,
          Register rm,
          Extend ext = UXTX,
          Scaling scale = Unscaled) {
    ASSERT((rn != R31) && (rn != ZR));
    ASSERT((rm != R31) && (rm != CSP));
    // Can only scale when ext = UXTX.
    ASSERT((scale != Scaled) || (ext == UXTX));
    ASSERT((ext == UXTW) || (ext == UXTX) || (ext == SXTW) || (ext == SXTX));
    const int32_t s = (scale == Scaled) ? B12 : 0;
    encoding_ = B21 | B11 | s | Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm) |
                (static_cast<int32_t>(ext) << kExtendTypeShift);
    type_ = Reg;
    base_ = ConcreteRegister(rn);
  }

  static OperandSize OperandSizeFor(intptr_t cid) {
    switch (cid) {
      case kArrayCid:
      case kImmutableArrayCid:
        return kWord;
      case kOneByteStringCid:
      case kExternalOneByteStringCid:
        return kByte;
      case kTwoByteStringCid:
      case kExternalTwoByteStringCid:
        return kHalfword;
      case kTypedDataInt8ArrayCid:
        return kByte;
      case kTypedDataUint8ArrayCid:
      case kTypedDataUint8ClampedArrayCid:
      case kExternalTypedDataUint8ArrayCid:
      case kExternalTypedDataUint8ClampedArrayCid:
        return kUnsignedByte;
      case kTypedDataInt16ArrayCid:
        return kHalfword;
      case kTypedDataUint16ArrayCid:
        return kUnsignedHalfword;
      case kTypedDataInt32ArrayCid:
        return kWord;
      case kTypedDataUint32ArrayCid:
        return kUnsignedWord;
      case kTypedDataInt64ArrayCid:
      case kTypedDataUint64ArrayCid:
        return kDWord;
      case kTypedDataFloat32ArrayCid:
        return kSWord;
      case kTypedDataFloat64ArrayCid:
        return kDWord;
      case kTypedDataFloat32x4ArrayCid:
      case kTypedDataInt32x4ArrayCid:
      case kTypedDataFloat64x2ArrayCid:
        return kQWord;
      case kTypedDataInt8ArrayViewCid:
        UNREACHABLE();
        return kByte;
      default:
        UNREACHABLE();
        return kByte;
    }
  }

 private:
  uint32_t encoding() const { return encoding_; }
  AddressType type() const { return type_; }
  Register base() const { return base_; }

  Address() : encoding_(0), type_(Unknown), base_(kNoRegister) {}

  uint32_t encoding_;
  AddressType type_;
  Register base_;

  friend class Assembler;
};

class FieldAddress : public Address {
 public:
  FieldAddress(Register base, int32_t disp, OperandSize sz = kDoubleWord)
      : Address(base, disp - kHeapObjectTag, Offset, sz) {}

  // This addressing mode does not exist.
  FieldAddress(Register base, Register disp, OperandSize sz = kDoubleWord);

  FieldAddress(const FieldAddress& other) : Address(other) {}

  FieldAddress& operator=(const FieldAddress& other) {
    Address::operator=(other);
    return *this;
  }
};

class Operand : public ValueObject {
 public:
  enum OperandType {
    Shifted,
    Extended,
    Immediate,
    BitfieldImm,
    Unknown,
  };

  // Data-processing operand - Uninitialized.
  Operand() : encoding_(-1), type_(Unknown) {}

  // Data-processing operands - Copy constructor.
  Operand(const Operand& other)
      : ValueObject(), encoding_(other.encoding_), type_(other.type_) {}

  Operand& operator=(const Operand& other) {
    type_ = other.type_;
    encoding_ = other.encoding_;
    return *this;
  }

  explicit Operand(Register rm) {
    ASSERT((rm != R31) && (rm != CSP));
    encoding_ = Arm64Encode::Rm(rm);
    type_ = Shifted;
  }

  Operand(Register rm, Shift shift, int32_t imm) {
    ASSERT(Utils::IsUint(6, imm));
    ASSERT((rm != R31) && (rm != CSP));
    encoding_ = (imm << kImm6Shift) | Arm64Encode::Rm(rm) |
                (static_cast<int32_t>(shift) << kShiftTypeShift);
    type_ = Shifted;
  }

  // This operand type does not exist.
  Operand(Register rm, Shift shift, Register r);

  Operand(Register rm, Extend extend, int32_t imm) {
    ASSERT(Utils::IsUint(3, imm));
    ASSERT((rm != R31) && (rm != CSP));
    encoding_ = B21 | Arm64Encode::Rm(rm) |
                (static_cast<int32_t>(extend) << kExtendTypeShift) |
                ((imm & 0x7) << kImm3Shift);
    type_ = Extended;
  }

  // This operand type does not exist.
  Operand(Register rm, Extend extend, Register r);

  explicit Operand(int32_t imm) {
    if (Utils::IsUint(12, imm)) {
      encoding_ = imm << kImm12Shift;
    } else {
      // imm only has bits in [12, 24) set.
      ASSERT(((imm & 0xfff) == 0) && (Utils::IsUint(12, imm >> 12)));
      encoding_ = B22 | ((imm >> 12) << kImm12Shift);
    }
    type_ = Immediate;
  }

  // Encodes the value of an immediate for a logical operation.
  // Since these values are difficult to craft by hand, instead pass the
  // logical mask to the function IsImmLogical to get n, imm_s, and
  // imm_r.  Takes s before r like DecodeBitMasks from Appendix G but unlike
  // the disassembly of the *bfm instructions.
  Operand(uint8_t n, int8_t imm_s, int8_t imm_r) {
    ASSERT((n == 1) || (n == 0));
    ASSERT(Utils::IsUint(6, imm_s) && Utils::IsUint(6, imm_r));
    type_ = BitfieldImm;
    encoding_ = (static_cast<int32_t>(n) << kNShift) |
                (static_cast<int32_t>(imm_s) << kImmSShift) |
                (static_cast<int32_t>(imm_r) << kImmRShift);
  }

  // Test if a given value can be encoded in the immediate field of a logical
  // instruction.
  // If it can be encoded, the function returns true, and values pointed to by
  // n, imm_s and imm_r are updated with immediates encoded in the format
  // required by the corresponding fields in the logical instruction.
  // If it can't be encoded, the function returns false, and the operand is
  // undefined.
  static bool IsImmLogical(uint64_t value, uint8_t width, Operand* imm_op);

  // An immediate imm can be an operand to add/sub when the return value is
  // Immediate, or a logical operation over sz bits when the return value is
  // BitfieldImm. If the return value is Unknown, then the immediate can't be
  // used as an operand in either instruction. The encoded operand is written
  // to op.
  static OperandType CanHold(int64_t imm, uint8_t sz, Operand* op) {
    ASSERT(op != NULL);
    ASSERT((sz == kXRegSizeInBits) || (sz == kWRegSizeInBits));
    if (Utils::IsUint(12, imm)) {
      op->encoding_ = imm << kImm12Shift;
      op->type_ = Immediate;
    } else if (((imm & 0xfff) == 0) && (Utils::IsUint(12, imm >> 12))) {
      op->encoding_ = B22 | ((imm >> 12) << kImm12Shift);
      op->type_ = Immediate;
    } else if (IsImmLogical(imm, sz, op)) {
      op->type_ = BitfieldImm;
    } else {
      op->encoding_ = 0;
      op->type_ = Unknown;
    }
    return op->type_;
  }

 private:
  uint32_t encoding() const { return encoding_; }
  OperandType type() const { return type_; }

  uint32_t encoding_;
  OperandType type_;

  friend class Assembler;
};

class Assembler : public AssemblerBase {
 public:
  explicit Assembler(ObjectPoolBuilder* object_pool_builder,
                     bool use_far_branches = false);
  ~Assembler() {}

  void PushRegister(Register r) { Push(r); }
  void PopRegister(Register r) { Pop(r); }

  void PushRegisterPair(Register r0, Register r1) { PushPair(r0, r1); }
  void PopRegisterPair(Register r0, Register r1) { PopPair(r0, r1); }

  void PushRegisters(const RegisterSet& registers);
  void PopRegisters(const RegisterSet& registers);

  // Push all registers which are callee-saved according to the ARM64 ABI.
  void PushNativeCalleeSavedRegisters();

  // Pop all registers which are callee-saved according to the ARM64 ABI.
  void PopNativeCalleeSavedRegisters();

  void MoveRegister(Register rd, Register rn) {
    if (rd != rn) {
      mov(rd, rn);
    }
  }

  void Drop(intptr_t stack_elements) {
    ASSERT(stack_elements >= 0);
    if (stack_elements > 0) {
      AddImmediate(SP, SP, stack_elements * target::kWordSize);
    }
  }

  void Bind(Label* label);
  void Jump(Label* label) { b(label); }

  void LoadField(Register dst, FieldAddress address) { ldr(dst, address); }
  void LoadMemoryValue(Register dst, Register base, int32_t offset) {
    LoadFromOffset(dst, base, offset, kDoubleWord);
  }
  void StoreMemoryValue(Register src, Register base, int32_t offset) {
    StoreToOffset(src, base, offset, kDoubleWord);
  }
  void LoadAcquire(Register dst, Register address, int32_t offset = 0) {
    if (offset != 0) {
      AddImmediate(TMP2, address, offset);
      ldar(dst, TMP2);
    } else {
      ldar(dst, address);
    }
  }
  void StoreRelease(Register src, Register address, int32_t offset = 0) {
    if (offset != 0) {
      AddImmediate(TMP2, address, offset);
      stlr(src, TMP2);
    } else {
      stlr(src, address);
    }
  }

  void CompareWithFieldValue(Register value, FieldAddress address) {
    CompareWithMemoryValue(value, address);
  }

  void CompareWithMemoryValue(Register value, Address address) {
    ldr(TMP, address);
    cmp(value, Operand(TMP));
  }

  void CompareTypeNullabilityWith(Register type, int8_t value) {
    ldr(TMP, FieldAddress(type, compiler::target::Type::nullability_offset()),
        kUnsignedByte);
    cmp(TMP, Operand(value));
  }

  bool use_far_branches() const {
    return FLAG_use_far_branches || use_far_branches_;
  }

  void set_use_far_branches(bool b) { use_far_branches_ = b; }

  // Debugging and bringup support.
  void Breakpoint() override { brk(0); }

  void SetPrologueOffset() {
    if (prologue_offset_ == -1) {
      prologue_offset_ = CodeSize();
    }
  }

  void ReserveAlignedFrameSpace(intptr_t frame_space);

  // In debug mode, this generates code to check that:
  //   FP + kExitLinkSlotFromEntryFp == SP
  // or triggers breakpoint otherwise.
  void EmitEntryFrameVerification();

  // Instruction pattern from entrypoint is used in Dart frame prologs
  // to set up the frame and save a PC which can be used to figure out the
  // RawInstruction object corresponding to the code running in the frame.
  static const intptr_t kEntryPointToPcMarkerOffset = 0;
  static intptr_t EntryPointToPcMarkerOffset() {
    return kEntryPointToPcMarkerOffset;
  }

  // Emit data (e.g encoded instruction or immediate) in instruction stream.
  void Emit(int32_t value);
  void Emit64(int64_t value);

  // On some other platforms, we draw a distinction between safe and unsafe
  // smis.
  static bool IsSafe(const Object& object) { return true; }
  static bool IsSafeSmi(const Object& object) { return target::IsSmi(object); }

  // Addition and subtraction.
  // For add and sub, to use CSP for rn, o must be of type Operand::Extend.
  // For an unmodified rm in this case, use Operand(rm, UXTX, 0);
  void add(Register rd, Register rn, Operand o) {
    AddSubHelper(kDoubleWord, false, false, rd, rn, o);
  }
  void adds(Register rd, Register rn, Operand o) {
    AddSubHelper(kDoubleWord, true, false, rd, rn, o);
  }
  void addw(Register rd, Register rn, Operand o) {
    AddSubHelper(kWord, false, false, rd, rn, o);
  }
  void addsw(Register rd, Register rn, Operand o) {
    AddSubHelper(kWord, true, false, rd, rn, o);
  }
  void sub(Register rd, Register rn, Operand o) {
    AddSubHelper(kDoubleWord, false, true, rd, rn, o);
  }
  void subs(Register rd, Register rn, Operand o) {
    AddSubHelper(kDoubleWord, true, true, rd, rn, o);
  }
  void subw(Register rd, Register rn, Operand o) {
    AddSubHelper(kWord, false, true, rd, rn, o);
  }
  void subsw(Register rd, Register rn, Operand o) {
    AddSubHelper(kWord, true, true, rd, rn, o);
  }

  // Addition and subtraction with carry.
  void adc(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kDoubleWord, false, false, rd, rn, rm);
  }
  void adcs(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kDoubleWord, true, false, rd, rn, rm);
  }
  void adcw(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kWord, false, false, rd, rn, rm);
  }
  void adcsw(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kWord, true, false, rd, rn, rm);
  }
  void sbc(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kDoubleWord, false, true, rd, rn, rm);
  }
  void sbcs(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kDoubleWord, true, true, rd, rn, rm);
  }
  void sbcw(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kWord, false, true, rd, rn, rm);
  }
  void sbcsw(Register rd, Register rn, Register rm) {
    AddSubWithCarryHelper(kWord, true, true, rd, rn, rm);
  }

  // PC relative immediate add. imm is in bytes.
  void adr(Register rd, const Immediate& imm) { EmitPCRelOp(ADR, rd, imm); }

  // Bitfield operations.
  // Bitfield move.
  // If s >= r then Rd[s-r:0] := Rn[s:r], else Rd[bitwidth+s-r:bitwidth-r] :=
  // Rn[s:0].
  void bfm(Register rd,
           Register rn,
           int r_imm,
           int s_imm,
           OperandSize size = kDoubleWord) {
    EmitBitfieldOp(BFM, rd, rn, r_imm, s_imm, size);
  }

  // Signed bitfield move.
  void sbfm(Register rd,
            Register rn,
            int r_imm,
            int s_imm,
            OperandSize size = kDoubleWord) {
    EmitBitfieldOp(SBFM, rd, rn, r_imm, s_imm, size);
  }

  // Unsigned bitfield move.
  void ubfm(Register rd,
            Register rn,
            int r_imm,
            int s_imm,
            OperandSize size = kDoubleWord) {
    EmitBitfieldOp(UBFM, rd, rn, r_imm, s_imm, size);
  }

  // Bitfield insert.  Takes the low width bits and replaces bits in rd with
  // them, starting at low_bit.
  void bfi(Register rd,
           Register rn,
           int low_bit,
           int width,
           OperandSize size = kDoubleWord) {
    int wordsize = size == kDoubleWord ? 64 : 32;
    EmitBitfieldOp(BFM, rd, rn, -low_bit & (wordsize - 1), width - 1, size);
  }

  // Bitfield extract and insert low.  Takes width bits, starting at low_bit and
  // replaces the low width bits of rd with them.
  void bfxil(Register rd,
             Register rn,
             int low_bit,
             int width,
             OperandSize size = kDoubleWord) {
    EmitBitfieldOp(BFM, rd, rn, low_bit, low_bit + width - 1, size);
  }

  // Signed bitfield insert in zero.  Takes the low width bits, sign extends
  // them and writes them to rd, starting at low_bit, and zeroing bits below
  // that.
  void sbfiz(Register rd,
             Register rn,
             int low_bit,
             int width,
             OperandSize size = kDoubleWord) {
    int wordsize = size == kDoubleWord ? 64 : 32;
    EmitBitfieldOp(SBFM, rd, rn, (wordsize - low_bit) & (wordsize - 1),
                   width - 1, size);
  }

  // Signed bitfield extract.  Takes width bits, starting at low_bit, sign
  // extends them and writes them to rd, starting at the lowest bit.
  void sbfx(Register rd,
            Register rn,
            int low_bit,
            int width,
            OperandSize size = kDoubleWord) {
    EmitBitfieldOp(SBFM, rd, rn, low_bit, low_bit + width - 1, size);
  }

  // Unsigned bitfield insert in zero.  Takes the low width bits and writes
  // them to rd, starting at low_bit, and zeroing bits above and below.
  void ubfiz(Register rd,
             Register rn,
             int low_bit,
             int width,
             OperandSize size = kDoubleWord) {
    int wordsize = size == kDoubleWord ? 64 : 32;
    ASSERT(width > 0);
    ASSERT(low_bit < wordsize);
    EmitBitfieldOp(UBFM, rd, rn, (-low_bit) & (wordsize - 1), width - 1, size);
  }

  // Unsigned bitfield extract.  Takes the width bits, starting at low_bit and
  // writes them to the low bits of rd zeroing bits above.
  void ubfx(Register rd,
            Register rn,
            int low_bit,
            int width,
            OperandSize size = kDoubleWord) {
    EmitBitfieldOp(UBFM, rd, rn, low_bit, low_bit + width - 1, size);
  }

  // Sign extend byte->64 bit.
  void sxtb(Register rd, Register rn) {
    EmitBitfieldOp(SBFM, rd, rn, 0, 7, kDoubleWord);
  }

  // Sign extend halfword->64 bit.
  void sxth(Register rd, Register rn) {
    EmitBitfieldOp(SBFM, rd, rn, 0, 15, kDoubleWord);
  }

  // Sign extend word->64 bit.
  void sxtw(Register rd, Register rn) {
    EmitBitfieldOp(SBFM, rd, rn, 0, 31, kDoubleWord);
  }

  // Zero/unsigned extend byte->64 bit.
  void uxtb(Register rd, Register rn) {
    EmitBitfieldOp(UBFM, rd, rn, 0, 7, kDoubleWord);
  }

  // Zero/unsigned extend halfword->64 bit.
  void uxth(Register rd, Register rn) {
    EmitBitfieldOp(UBFM, rd, rn, 0, 15, kDoubleWord);
  }

  // Zero/unsigned extend word->64 bit.
  void uxtw(Register rd, Register rn) {
    EmitBitfieldOp(UBFM, rd, rn, 0, 31, kDoubleWord);
  }

  // Logical immediate operations.
  void andi(Register rd, Register rn, const Immediate& imm) {
    Operand imm_op;
    const bool immok =
        Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
    ASSERT(immok);
    EmitLogicalImmOp(ANDI, rd, rn, imm_op, kDoubleWord);
  }
  void orri(Register rd, Register rn, const Immediate& imm) {
    Operand imm_op;
    const bool immok =
        Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
    ASSERT(immok);
    EmitLogicalImmOp(ORRI, rd, rn, imm_op, kDoubleWord);
  }
  void eori(Register rd, Register rn, const Immediate& imm) {
    Operand imm_op;
    const bool immok =
        Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
    ASSERT(immok);
    EmitLogicalImmOp(EORI, rd, rn, imm_op, kDoubleWord);
  }
  void andis(Register rd, Register rn, const Immediate& imm) {
    Operand imm_op;
    const bool immok =
        Operand::IsImmLogical(imm.value(), kXRegSizeInBits, &imm_op);
    ASSERT(immok);
    EmitLogicalImmOp(ANDIS, rd, rn, imm_op, kDoubleWord);
  }

  // Logical (shifted) register operations.
  void and_(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(AND, rd, rn, o, kDoubleWord);
  }
  void andw_(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(AND, rd, rn, o, kWord);
  }
  void bic(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(BIC, rd, rn, o, kDoubleWord);
  }
  void orr(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(ORR, rd, rn, o, kDoubleWord);
  }
  void orrw(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(ORR, rd, rn, o, kWord);
  }
  void orn(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(ORN, rd, rn, o, kDoubleWord);
  }
  void ornw(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(ORN, rd, rn, o, kWord);
  }
  void eor(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(EOR, rd, rn, o, kDoubleWord);
  }
  void eorw(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(EOR, rd, rn, o, kWord);
  }
  void eon(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(EON, rd, rn, o, kDoubleWord);
  }
  void ands(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(ANDS, rd, rn, o, kDoubleWord);
  }
  void bics(Register rd, Register rn, Operand o) {
    EmitLogicalShiftOp(BICS, rd, rn, o, kDoubleWord);
  }

  // Count leading zero bits.
  void clz(Register rd, Register rn) {
    EmitMiscDP1Source(CLZ, rd, rn, kDoubleWord);
  }

  // Reverse bits.
  void rbit(Register rd, Register rn) {
    EmitMiscDP1Source(RBIT, rd, rn, kDoubleWord);
  }

  // Misc. arithmetic.
  void udiv(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(UDIV, rd, rn, rm, kDoubleWord);
  }
  void sdiv(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(SDIV, rd, rn, rm, kDoubleWord);
  }
  void lslv(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(LSLV, rd, rn, rm, kDoubleWord);
  }
  void lsrv(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(LSRV, rd, rn, rm, kDoubleWord);
  }
  void asrv(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(ASRV, rd, rn, rm, kDoubleWord);
  }
  void lslvw(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(LSLV, rd, rn, rm, kWord);
  }
  void lsrvw(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(LSRV, rd, rn, rm, kWord);
  }
  void asrvw(Register rd, Register rn, Register rm) {
    EmitMiscDP2Source(ASRV, rd, rn, rm, kWord);
  }
  void madd(Register rd,
            Register rn,
            Register rm,
            Register ra,
            OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(MADD, rd, rn, rm, ra, sz);
  }
  void msub(Register rd,
            Register rn,
            Register rm,
            Register ra,
            OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(MSUB, rd, rn, rm, ra, sz);
  }
  void smulh(Register rd,
             Register rn,
             Register rm,
             OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(SMULH, rd, rn, rm, R31, sz);
  }
  void umulh(Register rd,
             Register rn,
             Register rm,
             OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(UMULH, rd, rn, rm, R31, sz);
  }
  void umaddl(Register rd,
              Register rn,
              Register rm,
              Register ra,
              OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(UMADDL, rd, rn, rm, ra, sz);
  }
  void umull(Register rd,
             Register rn,
             Register rm,
             OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(UMADDL, rd, rn, rm, ZR, sz);
  }
  void smaddl(Register rd,
              Register rn,
              Register rm,
              Register ra,
              OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(SMADDL, rd, rn, rm, ra, sz);
  }
  void smull(Register rd,
             Register rn,
             Register rm,
             OperandSize sz = kDoubleWord) {
    EmitMiscDP3Source(SMADDL, rd, rn, rm, ZR, sz);
  }

  // Move wide immediate.
  void movk(Register rd, const Immediate& imm, int hw_idx) {
    ASSERT(rd != CSP);
    const Register crd = ConcreteRegister(rd);
    EmitMoveWideOp(MOVK, crd, imm, hw_idx, kDoubleWord);
  }
  void movn(Register rd, const Immediate& imm, int hw_idx) {
    ASSERT(rd != CSP);
    const Register crd = ConcreteRegister(rd);
    EmitMoveWideOp(MOVN, crd, imm, hw_idx, kDoubleWord);
  }
  void movz(Register rd, const Immediate& imm, int hw_idx) {
    ASSERT(rd != CSP);
    const Register crd = ConcreteRegister(rd);
    EmitMoveWideOp(MOVZ, crd, imm, hw_idx, kDoubleWord);
  }

  // Loads and Stores.
  void ldr(Register rt, Address a, OperandSize sz = kDoubleWord) {
    ASSERT((a.type() != Address::PairOffset) &&
           (a.type() != Address::PairPostIndex) &&
           (a.type() != Address::PairPreIndex));
    if (a.type() == Address::PCOffset) {
      ASSERT(sz == kDoubleWord);
      EmitLoadRegLiteral(LDRpc, rt, a, sz);
    } else {
      if (IsSignedOperand(sz)) {
        EmitLoadStoreReg(LDRS, rt, a, sz);
      } else {
        EmitLoadStoreReg(LDR, rt, a, sz);
      }
    }
  }
  void str(Register rt, Address a, OperandSize sz = kDoubleWord) {
    ASSERT((a.type() != Address::PairOffset) &&
           (a.type() != Address::PairPostIndex) &&
           (a.type() != Address::PairPreIndex));
    EmitLoadStoreReg(STR, rt, a, sz);
  }

  void ldp(Register rt, Register rt2, Address a, OperandSize sz = kDoubleWord) {
    ASSERT((a.type() == Address::PairOffset) ||
           (a.type() == Address::PairPostIndex) ||
           (a.type() == Address::PairPreIndex));
    EmitLoadStoreRegPair(LDP, rt, rt2, a, sz);
  }
  void stp(Register rt, Register rt2, Address a, OperandSize sz = kDoubleWord) {
    ASSERT((a.type() == Address::PairOffset) ||
           (a.type() == Address::PairPostIndex) ||
           (a.type() == Address::PairPreIndex));
    EmitLoadStoreRegPair(STP, rt, rt2, a, sz);
  }

  void ldxr(Register rt, Register rn, OperandSize size = kDoubleWord) {
    // rt = value
    // rn = address
    EmitLoadStoreExclusive(LDXR, R31, rn, rt, size);
  }
  void stxr(Register rs,
            Register rt,
            Register rn,
            OperandSize size = kDoubleWord) {
    // rs = status (1 = failure, 0 = success)
    // rt = value
    // rn = address
    EmitLoadStoreExclusive(STXR, rs, rn, rt, size);
  }
  void clrex() {
    const int32_t encoding = static_cast<int32_t>(CLREX);
    Emit(encoding);
  }

  void ldar(Register rt, Register rn, OperandSize sz = kDoubleWord) {
    EmitLoadStoreExclusive(LDAR, R31, rn, rt, sz);
  }

  void stlr(Register rt, Register rn, OperandSize sz = kDoubleWord) {
    EmitLoadStoreExclusive(STLR, R31, rn, rt, sz);
  }

  // Conditional select.
  void csel(Register rd, Register rn, Register rm, Condition cond) {
    EmitConditionalSelect(CSEL, rd, rn, rm, cond, kDoubleWord);
  }
  void csinc(Register rd, Register rn, Register rm, Condition cond) {
    EmitConditionalSelect(CSINC, rd, rn, rm, cond, kDoubleWord);
  }
  void cinc(Register rd, Register rn, Condition cond) {
    csinc(rd, rn, rn, InvertCondition(cond));
  }
  void cset(Register rd, Condition cond) {
    csinc(rd, ZR, ZR, InvertCondition(cond));
  }
  void csinv(Register rd, Register rn, Register rm, Condition cond) {
    EmitConditionalSelect(CSINV, rd, rn, rm, cond, kDoubleWord);
  }
  void cinv(Register rd, Register rn, Condition cond) {
    csinv(rd, rn, rn, InvertCondition(cond));
  }
  void csetm(Register rd, Condition cond) {
    csinv(rd, ZR, ZR, InvertCondition(cond));
  }
  void csneg(Register rd, Register rn, Register rm, Condition cond) {
    EmitConditionalSelect(CSNEG, rd, rn, rm, cond, kDoubleWord);
  }
  void cneg(Register rd, Register rn, Condition cond) {
    EmitConditionalSelect(CSNEG, rd, rn, rn, InvertCondition(cond),
                          kDoubleWord);
  }

  // Comparison.
  // rn cmp o.
  // For add and sub, to use CSP for rn, o must be of type Operand::Extend.
  // For an unmodified rm in this case, use Operand(rm, UXTX, 0);
  void cmp(Register rn, Operand o) { subs(ZR, rn, o); }
  void cmpw(Register rn, Operand o) { subsw(ZR, rn, o); }
  // rn cmp -o.
  void cmn(Register rn, Operand o) { adds(ZR, rn, o); }

  void CompareRegisters(Register rn, Register rm) {
    if (rn == CSP) {
      // UXTX 0 on a 64-bit register (rm) is a nop, but forces R31 to be
      // interpreted as CSP.
      cmp(CSP, Operand(rm, UXTX, 0));
    } else {
      cmp(rn, Operand(rm));
    }
  }

  // Conditional branch.
  void b(Label* label, Condition cond = AL) {
    EmitConditionalBranch(BCOND, cond, label);
  }

  void b(int32_t offset) { EmitUnconditionalBranchOp(B, offset); }
  void bl(int32_t offset) { EmitUnconditionalBranchOp(BL, offset); }

  void BranchIf(Condition condition, Label* label) { b(label, condition); }

  void cbz(Label* label, Register rt, OperandSize sz = kDoubleWord) {
    EmitCompareAndBranch(CBZ, rt, label, sz);
  }

  void cbnz(Label* label, Register rt, OperandSize sz = kDoubleWord) {
    EmitCompareAndBranch(CBNZ, rt, label, sz);
  }

  // Generate 64-bit compare with zero and branch when condition allows to use
  // a single instruction: cbz/cbnz/tbz/tbnz.
  bool CanGenerateXCbzTbz(Register rn, Condition cond);
  void GenerateXCbzTbz(Register rn, Condition cond, Label* label);

  // Test bit and branch if zero.
  void tbz(Label* label, Register rt, intptr_t bit_number) {
    EmitTestAndBranch(TBZ, rt, bit_number, label);
  }
  void tbnz(Label* label, Register rt, intptr_t bit_number) {
    EmitTestAndBranch(TBNZ, rt, bit_number, label);
  }

  // Branch, link, return.
  void br(Register rn) { EmitUnconditionalBranchRegOp(BR, rn); }
  void blr(Register rn) { EmitUnconditionalBranchRegOp(BLR, rn); }
  void ret(Register rn = R30) { EmitUnconditionalBranchRegOp(RET, rn); }

  // Breakpoint.
  void brk(uint16_t imm) { EmitExceptionGenOp(BRK, imm); }

  // Double floating point.
  bool fmovdi(VRegister vd, double immd) {
    int64_t imm64 = bit_cast<int64_t, double>(immd);
    const uint8_t bit7 = imm64 >> 63;
    const uint8_t bit6 = (~(imm64 >> 62)) & 0x1;
    const uint8_t bit54 = (imm64 >> 52) & 0x3;
    const uint8_t bit30 = (imm64 >> 48) & 0xf;
    const uint8_t imm8 = (bit7 << 7) | (bit6 << 6) | (bit54 << 4) | bit30;
    const int64_t expimm8 = Instr::VFPExpandImm(imm8);
    if (imm64 != expimm8) {
      return false;
    }
    EmitFPImm(FMOVDI, vd, imm8);
    return true;
  }
  void fmovsr(VRegister vd, Register rn) {
    ASSERT(rn != R31);
    ASSERT(rn != CSP);
    const Register crn = ConcreteRegister(rn);
    EmitFPIntCvtOp(FMOVSR, static_cast<Register>(vd), crn, kWord);
  }
  void fmovrs(Register rd, VRegister vn) {
    ASSERT(rd != R31);
    ASSERT(rd != CSP);
    const Register crd = ConcreteRegister(rd);
    EmitFPIntCvtOp(FMOVRS, crd, static_cast<Register>(vn), kWord);
  }
  void fmovdr(VRegister vd, Register rn) {
    ASSERT(rn != R31);
    ASSERT(rn != CSP);
    const Register crn = ConcreteRegister(rn);
    EmitFPIntCvtOp(FMOVDR, static_cast<Register>(vd), crn);
  }
  void fmovrd(Register rd, VRegister vn) {
    ASSERT(rd != R31);
    ASSERT(rd != CSP);
    const Register crd = ConcreteRegister(rd);
    EmitFPIntCvtOp(FMOVRD, crd, static_cast<Register>(vn));
  }
  void scvtfdx(VRegister vd, Register rn) {
    ASSERT(rn != R31);
    ASSERT(rn != CSP);
    const Register crn = ConcreteRegister(rn);
    EmitFPIntCvtOp(SCVTFD, static_cast<Register>(vd), crn);
  }
  void scvtfdw(VRegister vd, Register rn) {
    ASSERT(rn != R31);
    ASSERT(rn != CSP);
    const Register crn = ConcreteRegister(rn);
    EmitFPIntCvtOp(SCVTFD, static_cast<Register>(vd), crn, kWord);
  }
  void fcvtzds(Register rd, VRegister vn) {
    ASSERT(rd != R31);
    ASSERT(rd != CSP);
    const Register crd = ConcreteRegister(rd);
    EmitFPIntCvtOp(FCVTZDS, crd, static_cast<Register>(vn));
  }
  void fmovdd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FMOVDD, vd, vn); }
  void fabsd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FABSD, vd, vn); }
  void fnegd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FNEGD, vd, vn); }
  void fsqrtd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FSQRTD, vd, vn); }
  void fcvtsd(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FCVTSD, vd, vn); }
  void fcvtds(VRegister vd, VRegister vn) { EmitFPOneSourceOp(FCVTDS, vd, vn); }
  void fldrq(VRegister vt, Address a) {
    ASSERT(a.type() != Address::PCOffset);
    EmitLoadStoreReg(FLDRQ, static_cast<Register>(vt), a, kByte);
  }
  void fstrq(VRegister vt, Address a) {
    ASSERT(a.type() != Address::PCOffset);
    EmitLoadStoreReg(FSTRQ, static_cast<Register>(vt), a, kByte);
  }
  void fldrd(VRegister vt, Address a) {
    ASSERT(a.type() != Address::PCOffset);
    EmitLoadStoreReg(FLDR, static_cast<Register>(vt), a, kDWord);
  }
  void fstrd(VRegister vt, Address a) {
    ASSERT(a.type() != Address::PCOffset);
    EmitLoadStoreReg(FSTR, static_cast<Register>(vt), a, kDWord);
  }
  void fldrs(VRegister vt, Address a) {
    ASSERT(a.type() != Address::PCOffset);
    EmitLoadStoreReg(FLDR, static_cast<Register>(vt), a, kSWord);
  }
  void fstrs(VRegister vt, Address a) {
    ASSERT(a.type() != Address::PCOffset);
    EmitLoadStoreReg(FSTR, static_cast<Register>(vt), a, kSWord);
  }
  void fcmpd(VRegister vn, VRegister vm) { EmitFPCompareOp(FCMPD, vn, vm); }
  void fcmpdz(VRegister vn) { EmitFPCompareOp(FCMPZD, vn, V0); }
  void fmuld(VRegister vd, VRegister vn, VRegister vm) {
    EmitFPTwoSourceOp(FMULD, vd, vn, vm);
  }
  void fdivd(VRegister vd, VRegister vn, VRegister vm) {
    EmitFPTwoSourceOp(FDIVD, vd, vn, vm);
  }
  void faddd(VRegister vd, VRegister vn, VRegister vm) {
    EmitFPTwoSourceOp(FADDD, vd, vn, vm);
  }
  void fsubd(VRegister vd, VRegister vn, VRegister vm) {
    EmitFPTwoSourceOp(FSUBD, vd, vn, vm);
  }

  // SIMD operations.
  void vand(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VAND, vd, vn, vm);
  }
  void vorr(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VORR, vd, vn, vm);
  }
  void veor(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VEOR, vd, vn, vm);
  }
  void vaddw(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VADDW, vd, vn, vm);
  }
  void vaddx(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VADDX, vd, vn, vm);
  }
  void vsubw(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VSUBW, vd, vn, vm);
  }
  void vsubx(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VSUBX, vd, vn, vm);
  }
  void vadds(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VADDS, vd, vn, vm);
  }
  void vaddd(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VADDD, vd, vn, vm);
  }
  void vsubs(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VSUBS, vd, vn, vm);
  }
  void vsubd(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VSUBD, vd, vn, vm);
  }
  void vmuls(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VMULS, vd, vn, vm);
  }
  void vmuld(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VMULD, vd, vn, vm);
  }
  void vdivs(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VDIVS, vd, vn, vm);
  }
  void vdivd(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VDIVD, vd, vn, vm);
  }
  void vceqs(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VCEQS, vd, vn, vm);
  }
  void vceqd(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VCEQD, vd, vn, vm);
  }
  void vcgts(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VCGTS, vd, vn, vm);
  }
  void vcgtd(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VCGTD, vd, vn, vm);
  }
  void vcges(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VCGES, vd, vn, vm);
  }
  void vcged(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VCGED, vd, vn, vm);
  }
  void vmins(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VMINS, vd, vn, vm);
  }
  void vmind(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VMIND, vd, vn, vm);
  }
  void vmaxs(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VMAXS, vd, vn, vm);
  }
  void vmaxd(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VMAXD, vd, vn, vm);
  }
  void vrecpss(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VRECPSS, vd, vn, vm);
  }
  void vrsqrtss(VRegister vd, VRegister vn, VRegister vm) {
    EmitSIMDThreeSameOp(VRSQRTSS, vd, vn, vm);
  }
  void vnot(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VNOT, vd, vn); }
  void vabss(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VABSS, vd, vn); }
  void vabsd(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VABSD, vd, vn); }
  void vnegs(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VNEGS, vd, vn); }
  void vnegd(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VNEGD, vd, vn); }
  void vsqrts(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VSQRTS, vd, vn); }
  void vsqrtd(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VSQRTD, vd, vn); }
  void vrecpes(VRegister vd, VRegister vn) {
    EmitSIMDTwoRegOp(VRECPES, vd, vn);
  }
  void vrsqrtes(VRegister vd, VRegister vn) {
    EmitSIMDTwoRegOp(VRSQRTES, vd, vn);
  }
  void vdupw(VRegister vd, Register rn) {
    const VRegister vn = static_cast<VRegister>(rn);
    EmitSIMDCopyOp(VDUPI, vd, vn, kWord, 0, 0);
  }
  void vdupx(VRegister vd, Register rn) {
    const VRegister vn = static_cast<VRegister>(rn);
    EmitSIMDCopyOp(VDUPI, vd, vn, kDoubleWord, 0, 0);
  }
  void vdups(VRegister vd, VRegister vn, int32_t idx) {
    EmitSIMDCopyOp(VDUP, vd, vn, kSWord, 0, idx);
  }
  void vdupd(VRegister vd, VRegister vn, int32_t idx) {
    EmitSIMDCopyOp(VDUP, vd, vn, kDWord, 0, idx);
  }
  void vinsw(VRegister vd, int32_t didx, Register rn) {
    const VRegister vn = static_cast<VRegister>(rn);
    EmitSIMDCopyOp(VINSI, vd, vn, kWord, 0, didx);
  }
  void vinsx(VRegister vd, int32_t didx, Register rn) {
    const VRegister vn = static_cast<VRegister>(rn);
    EmitSIMDCopyOp(VINSI, vd, vn, kDoubleWord, 0, didx);
  }
  void vinss(VRegister vd, int32_t didx, VRegister vn, int32_t sidx) {
    EmitSIMDCopyOp(VINS, vd, vn, kSWord, sidx, didx);
  }
  void vinsd(VRegister vd, int32_t didx, VRegister vn, int32_t sidx) {
    EmitSIMDCopyOp(VINS, vd, vn, kDWord, sidx, didx);
  }
  void vmovrs(Register rd, VRegister vn, int32_t sidx) {
    const VRegister vd = static_cast<VRegister>(rd);
    EmitSIMDCopyOp(VMOVW, vd, vn, kWord, 0, sidx);
  }
  void vmovrd(Register rd, VRegister vn, int32_t sidx) {
    const VRegister vd = static_cast<VRegister>(rd);
    EmitSIMDCopyOp(VMOVX, vd, vn, kDoubleWord, 0, sidx);
  }

  // Aliases.
  void mov(Register rd, Register rn) {
    if ((rd == CSP) || (rn == CSP)) {
      add(rd, rn, Operand(0));
    } else {
      orr(rd, ZR, Operand(rn));
    }
  }
  void movw(Register rd, Register rn) {
    if ((rd == CSP) || (rn == CSP)) {
      addw(rd, rn, Operand(0));
    } else {
      orrw(rd, ZR, Operand(rn));
    }
  }
  void vmov(VRegister vd, VRegister vn) { vorr(vd, vn, vn); }
  void mvn(Register rd, Register rm) { orn(rd, ZR, Operand(rm)); }
  void mvnw(Register rd, Register rm) { ornw(rd, ZR, Operand(rm)); }
  void neg(Register rd, Register rm) { sub(rd, ZR, Operand(rm)); }
  void negs(Register rd, Register rm) { subs(rd, ZR, Operand(rm)); }
  void negsw(Register rd, Register rm) { subsw(rd, ZR, Operand(rm)); }
  void mul(Register rd, Register rn, Register rm) {
    madd(rd, rn, rm, ZR, kDoubleWord);
  }
  void mulw(Register rd, Register rn, Register rm) {
    madd(rd, rn, rm, ZR, kWord);
  }
  void Push(Register reg) {
    ASSERT(reg != PP);  // Only push PP with TagAndPushPP().
    str(reg, Address(SP, -1 * target::kWordSize, Address::PreIndex));
  }
  void Pop(Register reg) {
    ASSERT(reg != PP);  // Only pop PP with PopAndUntagPP().
    ldr(reg, Address(SP, 1 * target::kWordSize, Address::PostIndex));
  }
  void PushPair(Register low, Register high) {
    stp(low, high, Address(SP, -2 * target::kWordSize, Address::PairPreIndex));
  }
  void PopPair(Register low, Register high) {
    ldp(low, high, Address(SP, 2 * target::kWordSize, Address::PairPostIndex));
  }
  void PushFloat(VRegister reg) {
    fstrs(reg, Address(SP, -1 * kFloatSize, Address::PreIndex));
  }
  void PushDouble(VRegister reg) {
    fstrd(reg, Address(SP, -1 * kDoubleSize, Address::PreIndex));
  }
  void PushQuad(VRegister reg) {
    fstrq(reg, Address(SP, -1 * kQuadSize, Address::PreIndex));
  }
  void PopFloat(VRegister reg) {
    fldrs(reg, Address(SP, 1 * kFloatSize, Address::PostIndex));
  }
  void PopDouble(VRegister reg) {
    fldrd(reg, Address(SP, 1 * kDoubleSize, Address::PostIndex));
  }
  void PopQuad(VRegister reg) {
    fldrq(reg, Address(SP, 1 * kQuadSize, Address::PostIndex));
  }
  void TagAndPushPP() {
    // Add the heap object tag back to PP before putting it on the stack.
    add(TMP, PP, Operand(kHeapObjectTag));
    str(TMP, Address(SP, -1 * target::kWordSize, Address::PreIndex));
  }
  void TagAndPushPPAndPcMarker() {
    COMPILE_ASSERT(CODE_REG != TMP2);
    // Add the heap object tag back to PP before putting it on the stack.
    add(TMP2, PP, Operand(kHeapObjectTag));
    stp(TMP2, CODE_REG,
        Address(SP, -2 * target::kWordSize, Address::PairPreIndex));
  }
  void PopAndUntagPP() {
    ldr(PP, Address(SP, 1 * target::kWordSize, Address::PostIndex));
    sub(PP, PP, Operand(kHeapObjectTag));
    // The caller of PopAndUntagPP() must explicitly allow use of popped PP.
    set_constant_pool_allowed(false);
  }
  void tst(Register rn, Operand o) { ands(ZR, rn, o); }
  void tsti(Register rn, const Immediate& imm) { andis(ZR, rn, imm); }

  void LslImmediate(Register rd,
                    Register rn,
                    int shift,
                    OperandSize sz = kDoubleWord) {
    const int reg_size =
        (sz == kDoubleWord) ? kXRegSizeInBits : kWRegSizeInBits;
    ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1, sz);
  }
  void LsrImmediate(Register rd,
                    Register rn,
                    int shift,
                    OperandSize sz = kDoubleWord) {
    const int reg_size =
        (sz == kDoubleWord) ? kXRegSizeInBits : kWRegSizeInBits;
    ubfm(rd, rn, shift, reg_size - 1, sz);
  }
  void AsrImmediate(Register rd, Register rn, int shift) {
    const int reg_size = kXRegSizeInBits;
    sbfm(rd, rn, shift, reg_size - 1);
  }

  void VRecps(VRegister vd, VRegister vn);
  void VRSqrts(VRegister vd, VRegister vn);

  void SmiUntag(Register reg) { AsrImmediate(reg, reg, kSmiTagSize); }
  void SmiUntag(Register dst, Register src) {
    AsrImmediate(dst, src, kSmiTagSize);
  }
  void SmiTag(Register reg) { LslImmediate(reg, reg, kSmiTagSize); }
  void SmiTag(Register dst, Register src) {
    LslImmediate(dst, src, kSmiTagSize);
  }

  void BranchIfNotSmi(Register reg, Label* label) { tbnz(label, reg, kSmiTag); }

  void BranchIfSmi(Register reg, Label* label) { tbz(label, reg, kSmiTag); }

  void Branch(const Code& code,
              Register pp,
              ObjectPoolBuilderEntry::Patchability patchable =
                  ObjectPoolBuilderEntry::kNotPatchable);

  void BranchLink(const Code& code,
                  ObjectPoolBuilderEntry::Patchability patchable =
                      ObjectPoolBuilderEntry::kNotPatchable,
                  CodeEntryKind entry_kind = CodeEntryKind::kNormal);

  void BranchLinkPatchable(const Code& code,
                           CodeEntryKind entry_kind = CodeEntryKind::kNormal) {
    BranchLink(code, ObjectPoolBuilderEntry::kPatchable, entry_kind);
  }
  void BranchLinkToRuntime();

  // Emit a call that shares its object pool entries with other calls
  // that have the same equivalence marker.
  void BranchLinkWithEquivalence(
      const Code& code,
      const Object& equivalence,
      CodeEntryKind entry_kind = CodeEntryKind::kNormal);

  void Call(Address target) {
    ldr(LR, target);
    blr(LR);
  }

  void CallCFunction(Address target) { Call(target); }

  void AddImmediate(Register dest, int64_t imm) {
    AddImmediate(dest, dest, imm);
  }

  // Macros accepting a pp Register argument may attempt to load values from
  // the object pool when possible. Unless you are sure that the untagged object
  // pool pointer is in another register, or that it is not available at all,
  // PP should be passed for pp. `dest` can be TMP2, `rn` cannot.
  void AddImmediate(Register dest, Register rn, int64_t imm);
  void AddImmediateSetFlags(Register dest,
                            Register rn,
                            int64_t imm,
                            OperandSize sz = kDoubleWord);
  void SubImmediateSetFlags(Register dest,
                            Register rn,
                            int64_t imm,
                            OperandSize sz = kDoubleWord);
  void AndImmediate(Register rd, Register rn, int64_t imm);
  void OrImmediate(Register rd, Register rn, int64_t imm);
  void XorImmediate(Register rd, Register rn, int64_t imm);
  void TestImmediate(Register rn, int64_t imm);
  void CompareImmediate(Register rn, int64_t imm);

  void LoadFromOffset(Register dest,
                      Register base,
                      int32_t offset,
                      OperandSize sz = kDoubleWord);
  void LoadFieldFromOffset(Register dest,
                           Register base,
                           int32_t offset,
                           OperandSize sz = kDoubleWord) {
    LoadFromOffset(dest, base, offset - kHeapObjectTag, sz);
  }
  void LoadSFromOffset(VRegister dest, Register base, int32_t offset);
  void LoadDFromOffset(VRegister dest, Register base, int32_t offset);
  void LoadDFieldFromOffset(VRegister dest, Register base, int32_t offset) {
    LoadDFromOffset(dest, base, offset - kHeapObjectTag);
  }
  void LoadQFromOffset(VRegister dest, Register base, int32_t offset);
  void LoadQFieldFromOffset(VRegister dest, Register base, int32_t offset) {
    LoadQFromOffset(dest, base, offset - kHeapObjectTag);
  }

  void StoreToOffset(Register src,
                     Register base,
                     int32_t offset,
                     OperandSize sz = kDoubleWord);
  void StoreFieldToOffset(Register src,
                          Register base,
                          int32_t offset,
                          OperandSize sz = kDoubleWord) {
    StoreToOffset(src, base, offset - kHeapObjectTag, sz);
  }

  void StoreSToOffset(VRegister src, Register base, int32_t offset);
  void StoreDToOffset(VRegister src, Register base, int32_t offset);
  void StoreDFieldToOffset(VRegister src, Register base, int32_t offset) {
    StoreDToOffset(src, base, offset - kHeapObjectTag);
  }
  void StoreQToOffset(VRegister src, Register base, int32_t offset);
  void StoreQFieldToOffset(VRegister src, Register base, int32_t offset) {
    StoreQToOffset(src, base, offset - kHeapObjectTag);
  }

  enum CanBeSmi {
    kValueIsNotSmi,
    kValueCanBeSmi,
  };

  // Store into a heap object and apply the generational and incremental write
  // barriers. All stores into heap objects must pass through this function or,
  // if the value can be proven either Smi or old-and-premarked, its NoBarrier
  // variants.
  // Preserves object and value registers.
  void StoreIntoObject(Register object,
                       const Address& dest,
                       Register value,
                       CanBeSmi can_value_be_smi = kValueCanBeSmi,
                       bool lr_reserved = false);
  void StoreIntoArray(Register object,
                      Register slot,
                      Register value,
                      CanBeSmi can_value_be_smi = kValueCanBeSmi,
                      bool lr_reserved = false);

  void StoreIntoObjectOffset(Register object,
                             int32_t offset,
                             Register value,
                             CanBeSmi can_value_be_smi = kValueCanBeSmi,
                             bool lr_reserved = false);
  void StoreIntoObjectNoBarrier(Register object,
                                const Address& dest,
                                Register value);
  void StoreIntoObjectOffsetNoBarrier(Register object,
                                      int32_t offset,
                                      Register value);
  void StoreIntoObjectNoBarrier(Register object,
                                const Address& dest,
                                const Object& value);
  void StoreIntoObjectOffsetNoBarrier(Register object,
                                      int32_t offset,
                                      const Object& value);

  // Stores a non-tagged value into a heap object.
  void StoreInternalPointer(Register object,
                            const Address& dest,
                            Register value);

  // Object pool, loading from pool, etc.
  void LoadPoolPointer(Register pp = PP);

  bool constant_pool_allowed() const { return constant_pool_allowed_; }
  void set_constant_pool_allowed(bool b) { constant_pool_allowed_ = b; }

  intptr_t FindImmediate(int64_t imm);
  bool CanLoadFromObjectPool(const Object& object) const;
  void LoadNativeEntry(Register dst,
                       const ExternalLabel* label,
                       ObjectPoolBuilderEntry::Patchability patchable);
  void LoadIsolate(Register dst);

  // Note: the function never clobbers TMP, TMP2 scratch registers.
  void LoadObject(Register dst, const Object& obj);
  // Note: the function never clobbers TMP, TMP2 scratch registers.
  void LoadUniqueObject(Register dst, const Object& obj);
  // Note: the function never clobbers TMP, TMP2 scratch registers.
  void LoadImmediate(Register reg, int64_t imm);

  void LoadDImmediate(VRegister reg, double immd);

  // Load word from pool from the given offset using encoding that
  // InstructionPattern::DecodeLoadWordFromPool can decode.
  //
  // Note: the function never clobbers TMP, TMP2 scratch registers.
  void LoadWordFromPoolOffset(Register dst, uint32_t offset, Register pp = PP);

  void LoadDoubleWordFromPoolOffset(Register lower,
                                    Register upper,
                                    uint32_t offset);

  void PushObject(const Object& object) {
    if (IsSameObject(compiler::NullObject(), object)) {
      Push(NULL_REG);
    } else {
      LoadObject(TMP, object);
      Push(TMP);
    }
  }
  void PushImmediate(int64_t immediate) {
    LoadImmediate(TMP, immediate);
    Push(TMP);
  }
  void CompareObject(Register reg, const Object& object);

  void ExtractClassIdFromTags(Register result, Register tags);
  void ExtractInstanceSizeFromTags(Register result, Register tags);

  void LoadClassId(Register result, Register object);
  void LoadClassById(Register result, Register class_id);
  void CompareClassId(Register object,
                      intptr_t class_id,
                      Register scratch = kNoRegister);
  // Note: input and output registers must be different.
  void LoadClassIdMayBeSmi(Register result, Register object);
  void LoadTaggedClassIdMayBeSmi(Register result, Register object);

  // Reserve specifies how much space to reserve for the Dart stack.
  void SetupDartSP(intptr_t reserve = 4096);
  void SetupCSPFromThread(Register thr);
  void RestoreCSP();

  void EnterFrame(intptr_t frame_size);
  void LeaveFrame();
  void Ret() { ret(LR); }

  // Emit code to transition between generated mode and native mode.
  //
  // These require and ensure that CSP and SP are equal and aligned and require
  // a scratch register (in addition to TMP/TMP2).

  void TransitionGeneratedToNative(Register destination_address,
                                   Register new_exit_frame,
                                   Register new_exit_through_ffi,
                                   bool enter_safepoint);
  void TransitionNativeToGenerated(Register scratch, bool exit_safepoint);
  void EnterSafepoint(Register scratch);
  void ExitSafepoint(Register scratch);

  void CheckCodePointer();
  void RestoreCodePointer();

  // Restores the values of the registers that are blocked to cache some values
  // e.g. BARRIER_MASK and NULL_REG.
  void RestorePinnedRegisters();

  void SetupGlobalPoolAndDispatchTable();

  void EnterDartFrame(intptr_t frame_size, Register new_pp = kNoRegister);
  void EnterOsrFrame(intptr_t extra_size, Register new_pp = kNoRegister);
  void LeaveDartFrame(RestorePP restore_pp = kRestoreCallerPP);

  void EnterCallRuntimeFrame(intptr_t frame_size);
  void LeaveCallRuntimeFrame();
  void CallRuntime(const RuntimeEntry& entry, intptr_t argument_count);

  // Set up a stub frame so that the stack traversal code can easily identify
  // a stub frame.
  void EnterStubFrame();
  void LeaveStubFrame();

  // Set up a frame for calling a C function.
  // Automatically save the pinned registers in Dart which are not callee-
  // saved in the native calling convention.
  // Use together with CallCFunction.
  void EnterCFrame(intptr_t frame_space);
  void LeaveCFrame();

  void MonomorphicCheckedEntryJIT();
  void MonomorphicCheckedEntryAOT();
  void BranchOnMonomorphicCheckedEntryJIT(Label* label);

  // If allocation tracing for |cid| is enabled, will jump to |trace| label,
  // which will allocate in the runtime where tracing occurs.
  void MaybeTraceAllocation(intptr_t cid, Register temp_reg, Label* trace);

  // Inlined allocation of an instance of class 'cls', code has no runtime
  // calls. Jump to 'failure' if the instance cannot be allocated here.
  // Allocated instance is returned in 'instance_reg'.
  // Only the tags field of the object is initialized.
  // Result:
  //   * [instance_reg] will contain allocated new-space object
  //   * [top_reg] will contain Thread::top_offset()
  void TryAllocate(const Class& cls,
                   Label* failure,
                   Register instance_reg,
                   Register top_reg,
                   bool tag_result = true);

  void TryAllocateArray(intptr_t cid,
                        intptr_t instance_size,
                        Label* failure,
                        Register instance,
                        Register end_address,
                        Register temp1,
                        Register temp2);

  // This emits an PC-relative call of the form "bl <offset>".  The offset
  // is not yet known and needs therefore relocation to the right place before
  // the code can be used.
  //
  // The neccessary information for the "linker" (i.e. the relocation
  // information) is stored in [CodeLayout::static_calls_target_table_]: an
  // entry of the form
  //
  //   (Code::kPcRelativeCall & pc_offset, <target-code>, <target-function>)
  //
  // will be used during relocation to fix the offset.
  //
  // The provided [offset_into_target] will be added to calculate the final
  // destination.  It can be used e.g. for calling into the middle of a
  // function.
  void GenerateUnRelocatedPcRelativeCall(intptr_t offset_into_target = 0);

  // This emits an PC-relative tail call of the form "b <offset>".
  //
  // See also above for the pc-relative call.
  void GenerateUnRelocatedPcRelativeTailCall(intptr_t offset_into_target = 0);

  Address ElementAddressForIntIndex(bool is_external,
                                    intptr_t cid,
                                    intptr_t index_scale,
                                    Register array,
                                    intptr_t index) const;
  void ComputeElementAddressForIntIndex(Register address,
                                        bool is_external,
                                        intptr_t cid,
                                        intptr_t index_scale,
                                        Register array,
                                        intptr_t index);
  Address ElementAddressForRegIndex(bool is_external,
                                    intptr_t cid,
                                    intptr_t index_scale,
                                    bool index_unboxed,
                                    Register array,
                                    Register index,
                                    Register temp);

  // Special version of ElementAddressForRegIndex for the case when cid and
  // operand size for the target load don't match (e.g. when loading a few
  // elements of the array with one load).
  Address ElementAddressForRegIndexWithSize(bool is_external,
                                            intptr_t cid,
                                            OperandSize size,
                                            intptr_t index_scale,
                                            bool index_unboxed,
                                            Register array,
                                            Register index,
                                            Register temp);

  void ComputeElementAddressForRegIndex(Register address,
                                        bool is_external,
                                        intptr_t cid,
                                        intptr_t index_scale,
                                        bool index_unboxed,
                                        Register array,
                                        Register index);

  void LoadFieldAddressForRegOffset(Register address,
                                    Register instance,
                                    Register offset_in_words_as_smi);

  // Returns object data offset for address calculation; for heap objects also
  // accounts for the tag.
  static int32_t HeapDataOffset(bool is_external, intptr_t cid) {
    return is_external
               ? 0
               : (target::Instance::DataOffsetFor(cid) - kHeapObjectTag);
  }

  static int32_t EncodeImm26BranchOffset(int64_t imm, int32_t instr) {
    const int32_t imm32 = static_cast<int32_t>(imm);
    const int32_t off = (((imm32 >> 2) << kImm26Shift) & kImm26Mask);
    return (instr & ~kImm26Mask) | off;
  }

  static int64_t DecodeImm26BranchOffset(int32_t instr) {
    const int32_t off = (((instr & kImm26Mask) >> kImm26Shift) << 6) >> 4;
    return static_cast<int64_t>(off);
  }

 private:
  bool use_far_branches_;

  bool constant_pool_allowed_;

  void LoadWordFromPoolOffsetFixed(Register dst, uint32_t offset);

  // Note: the function never clobbers TMP, TMP2 scratch registers.
  void LoadObjectHelper(Register dst, const Object& obj, bool is_unique);

  void AddSubHelper(OperandSize os,
                    bool set_flags,
                    bool subtract,
                    Register rd,
                    Register rn,
                    Operand o) {
    ASSERT((rd != R31) && (rn != R31));
    const Register crd = ConcreteRegister(rd);
    const Register crn = ConcreteRegister(rn);
    if (o.type() == Operand::Immediate) {
      ASSERT(rn != ZR);
      EmitAddSubImmOp(subtract ? SUBI : ADDI, crd, crn, o, os, set_flags);
    } else if (o.type() == Operand::Shifted) {
      ASSERT((rd != CSP) && (rn != CSP));
      EmitAddSubShiftExtOp(subtract ? SUB : ADD, crd, crn, o, os, set_flags);
    } else {
      ASSERT(o.type() == Operand::Extended);
      ASSERT((rd != CSP) && (rn != ZR));
      EmitAddSubShiftExtOp(subtract ? SUB : ADD, crd, crn, o, os, set_flags);
    }
  }

  void AddSubWithCarryHelper(OperandSize sz,
                             bool set_flags,
                             bool subtract,
                             Register rd,
                             Register rn,
                             Register rm) {
    ASSERT((rd != R31) && (rn != R31) && (rm != R31));
    ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t s = set_flags ? B29 : 0;
    const int32_t op = subtract ? SBC : ADC;
    const int32_t encoding = op | size | s | Arm64Encode::Rd(rd) |
                             Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm);
    Emit(encoding);
  }

  void EmitAddSubImmOp(AddSubImmOp op,
                       Register rd,
                       Register rn,
                       Operand o,
                       OperandSize sz,
                       bool set_flags) {
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t s = set_flags ? B29 : 0;
    const int32_t encoding = op | size | s | Arm64Encode::Rd(rd) |
                             Arm64Encode::Rn(rn) | o.encoding();
    Emit(encoding);
  }

  // Follows the *bfm instructions in taking r before s (unlike the Operand
  // constructor, which follows DecodeBitMasks from Appendix G).
  void EmitBitfieldOp(BitfieldOp op,
                      Register rd,
                      Register rn,
                      int r_imm,
                      int s_imm,
                      OperandSize size) {
    if (size != kDoubleWord) {
      ASSERT(size == kWord);
      ASSERT(r_imm < 32 && s_imm < 32);
    } else {
      ASSERT(r_imm < 64 && s_imm < 64);
    }
    const int32_t instr = op | (size == kDoubleWord ? Bitfield64 : 0);
    const int32_t encoding = instr | Operand(0, s_imm, r_imm).encoding() |
                             Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn);
    Emit(encoding);
  }

  void EmitLogicalImmOp(LogicalImmOp op,
                        Register rd,
                        Register rn,
                        Operand o,
                        OperandSize sz) {
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    ASSERT((rd != R31) && (rn != R31));
    ASSERT(rn != CSP);
    ASSERT((op == ANDIS) || (rd != ZR));   // op != ANDIS => rd != ZR.
    ASSERT((op != ANDIS) || (rd != CSP));  // op == ANDIS => rd != CSP.
    ASSERT(o.type() == Operand::BitfieldImm);
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding =
        op | size | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn) | o.encoding();
    Emit(encoding);
  }

  void EmitLogicalShiftOp(LogicalShiftOp op,
                          Register rd,
                          Register rn,
                          Operand o,
                          OperandSize sz) {
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    ASSERT((rd != R31) && (rn != R31));
    ASSERT((rd != CSP) && (rn != CSP));
    ASSERT(o.type() == Operand::Shifted);
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding =
        op | size | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn) | o.encoding();
    Emit(encoding);
  }

  void EmitAddSubShiftExtOp(AddSubShiftExtOp op,
                            Register rd,
                            Register rn,
                            Operand o,
                            OperandSize sz,
                            bool set_flags) {
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t s = set_flags ? B29 : 0;
    const int32_t encoding = op | size | s | Arm64Encode::Rd(rd) |
                             Arm64Encode::Rn(rn) | o.encoding();
    Emit(encoding);
  }

  int32_t BindImm19Branch(int64_t position, int64_t dest);
  int32_t BindImm14Branch(int64_t position, int64_t dest);

  int32_t EncodeImm19BranchOffset(int64_t imm, int32_t instr) {
    if (!CanEncodeImm19BranchOffset(imm)) {
      ASSERT(!use_far_branches());
      BailoutWithBranchOffsetError();
    }
    const int32_t imm32 = static_cast<int32_t>(imm);
    const int32_t off =
        ((static_cast<uint32_t>(imm32 >> 2) << kImm19Shift) & kImm19Mask);
    return (instr & ~kImm19Mask) | off;
  }

  int64_t DecodeImm19BranchOffset(int32_t instr) {
    int32_t insns = (static_cast<uint32_t>(instr) & kImm19Mask) >> kImm19Shift;
    const int32_t off = static_cast<int32_t>(insns << 13) >> 11;
    return static_cast<int64_t>(off);
  }

  int32_t EncodeImm14BranchOffset(int64_t imm, int32_t instr) {
    if (!CanEncodeImm14BranchOffset(imm)) {
      ASSERT(!use_far_branches());
      BailoutWithBranchOffsetError();
    }
    const int32_t imm32 = static_cast<int32_t>(imm);
    const int32_t off =
        ((static_cast<uint32_t>(imm32 >> 2) << kImm14Shift) & kImm14Mask);
    return (instr & ~kImm14Mask) | off;
  }

  int64_t DecodeImm14BranchOffset(int32_t instr) {
    int32_t insns = (static_cast<uint32_t>(instr) & kImm14Mask) >> kImm14Shift;
    const int32_t off = static_cast<int32_t>(insns << 18) >> 16;
    return static_cast<int64_t>(off);
  }

  bool IsConditionalBranch(int32_t instr) {
    return (instr & ConditionalBranchMask) ==
           (ConditionalBranchFixed & ConditionalBranchMask);
  }

  bool IsCompareAndBranch(int32_t instr) {
    return (instr & CompareAndBranchMask) ==
           (CompareAndBranchFixed & CompareAndBranchMask);
  }

  bool IsTestAndBranch(int32_t instr) {
    return (instr & TestAndBranchMask) ==
           (TestAndBranchFixed & TestAndBranchMask);
  }

  Condition DecodeImm19BranchCondition(int32_t instr) {
    if (IsConditionalBranch(instr)) {
      return static_cast<Condition>((instr & kCondMask) >> kCondShift);
    }
    ASSERT(IsCompareAndBranch(instr));
    return (instr & B24) ? EQ : NE;  // cbz : cbnz
  }

  int32_t EncodeImm19BranchCondition(Condition cond, int32_t instr) {
    if (IsConditionalBranch(instr)) {
      const int32_t c_imm = static_cast<int32_t>(cond);
      return (instr & ~kCondMask) | (c_imm << kCondShift);
    }
    ASSERT(IsCompareAndBranch(instr));
    return (instr & ~B24) | (cond == EQ ? B24 : 0);  // cbz : cbnz
  }

  Condition DecodeImm14BranchCondition(int32_t instr) {
    ASSERT(IsTestAndBranch(instr));
    return (instr & B24) ? EQ : NE;  // tbz : tbnz
  }

  int32_t EncodeImm14BranchCondition(Condition cond, int32_t instr) {
    ASSERT(IsTestAndBranch(instr));
    return (instr & ~B24) | (cond == EQ ? B24 : 0);  // tbz : tbnz
  }

  void EmitCompareAndBranchOp(CompareAndBranchOp op,
                              Register rt,
                              int64_t imm,
                              OperandSize sz) {
    // EncodeImm19BranchOffset will longjump out if the offset does not fit in
    // 19 bits.
    const int32_t encoded_offset = EncodeImm19BranchOffset(imm, 0);
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    ASSERT(Utils::IsInt(21, imm) && ((imm & 0x3) == 0));
    ASSERT((rt != CSP) && (rt != R31));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding = op | size | Arm64Encode::Rt(rt) | encoded_offset;
    Emit(encoding);
  }

  void EmitTestAndBranchOp(TestAndBranchOp op,
                           Register rt,
                           intptr_t bit_number,
                           int64_t imm) {
    // EncodeImm14BranchOffset will longjump out if the offset does not fit in
    // 14 bits.
    const int32_t encoded_offset = EncodeImm14BranchOffset(imm, 0);
    ASSERT((bit_number >= 0) && (bit_number <= 63));
    ASSERT(Utils::IsInt(16, imm) && ((imm & 0x3) == 0));
    ASSERT((rt != CSP) && (rt != R31));
    const Register crt = ConcreteRegister(rt);
    int32_t bit_number_low = bit_number & 0x1f;
    int32_t bit_number_hi = (bit_number & 0x20) >> 5;
    const int32_t encoding =
        op | (bit_number_low << 19) | (bit_number_hi << 31) |
        (static_cast<int32_t>(crt) << kRtShift) | encoded_offset;
    Emit(encoding);
  }

  void EmitConditionalBranchOp(ConditionalBranchOp op,
                               Condition cond,
                               int64_t imm) {
    const int32_t off = EncodeImm19BranchOffset(imm, 0);
    const int32_t encoding =
        op | (static_cast<int32_t>(cond) << kCondShift) | off;
    Emit(encoding);
  }

  bool CanEncodeImm19BranchOffset(int64_t offset) {
    ASSERT(Utils::IsAligned(offset, 4));
    return Utils::IsInt(21, offset);
  }

  bool CanEncodeImm14BranchOffset(int64_t offset) {
    ASSERT(Utils::IsAligned(offset, 4));
    return Utils::IsInt(16, offset);
  }

  void EmitConditionalBranch(ConditionalBranchOp op,
                             Condition cond,
                             Label* label) {
    if (label->IsBound()) {
      const int64_t dest = label->Position() - buffer_.Size();
      if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) {
        if (cond == AL) {
          // If the condition is AL, we must always branch to dest. There is
          // no need for a guard branch.
          b(dest);
        } else {
          EmitConditionalBranchOp(op, InvertCondition(cond),
                                  2 * Instr::kInstrSize);
          // Make a new dest that takes the new position into account after the
          // inverted test.
          const int64_t dest = label->Position() - buffer_.Size();
          b(dest);
        }
      } else {
        EmitConditionalBranchOp(op, cond, dest);
      }
    } else {
      const int64_t position = buffer_.Size();
      if (use_far_branches()) {
        // When cond is AL, this guard branch will be rewritten as a nop when
        // the label is bound. We don't write it as a nop initially because it
        // makes the decoding code in Bind simpler.
        EmitConditionalBranchOp(op, InvertCondition(cond),
                                2 * Instr::kInstrSize);
        b(label->position_);
      } else {
        EmitConditionalBranchOp(op, cond, label->position_);
      }
      label->LinkTo(position);
    }
  }

  void EmitCompareAndBranch(CompareAndBranchOp op,
                            Register rt,
                            Label* label,
                            OperandSize sz) {
    if (label->IsBound()) {
      const int64_t dest = label->Position() - buffer_.Size();
      if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) {
        EmitCompareAndBranchOp(op == CBZ ? CBNZ : CBZ, rt,
                               2 * Instr::kInstrSize, sz);
        // Make a new dest that takes the new position into account after the
        // inverted test.
        const int64_t dest = label->Position() - buffer_.Size();
        b(dest);
      } else {
        EmitCompareAndBranchOp(op, rt, dest, sz);
      }
    } else {
      const int64_t position = buffer_.Size();
      if (use_far_branches()) {
        EmitCompareAndBranchOp(op == CBZ ? CBNZ : CBZ, rt,
                               2 * Instr::kInstrSize, sz);
        b(label->position_);
      } else {
        EmitCompareAndBranchOp(op, rt, label->position_, sz);
      }
      label->LinkTo(position);
    }
  }

  void EmitTestAndBranch(TestAndBranchOp op,
                         Register rt,
                         intptr_t bit_number,
                         Label* label) {
    if (label->IsBound()) {
      const int64_t dest = label->Position() - buffer_.Size();
      if (use_far_branches() && !CanEncodeImm14BranchOffset(dest)) {
        EmitTestAndBranchOp(op == TBZ ? TBNZ : TBZ, rt, bit_number,
                            2 * Instr::kInstrSize);
        // Make a new dest that takes the new position into account after the
        // inverted test.
        const int64_t dest = label->Position() - buffer_.Size();
        b(dest);
      } else {
        EmitTestAndBranchOp(op, rt, bit_number, dest);
      }
    } else {
      int64_t position = buffer_.Size();
      if (use_far_branches()) {
        EmitTestAndBranchOp(op == TBZ ? TBNZ : TBZ, rt, bit_number,
                            2 * Instr::kInstrSize);
        b(label->position_);
      } else {
        EmitTestAndBranchOp(op, rt, bit_number, label->position_);
      }
      label->LinkTo(position);
    }
  }

  bool CanEncodeImm26BranchOffset(int64_t offset) {
    ASSERT(Utils::IsAligned(offset, 4));
    return Utils::IsInt(26, offset);
  }

  void EmitUnconditionalBranchOp(UnconditionalBranchOp op, int64_t offset) {
    ASSERT(CanEncodeImm26BranchOffset(offset));
    const int32_t off = ((offset >> 2) << kImm26Shift) & kImm26Mask;
    const int32_t encoding = op | off;
    Emit(encoding);
  }

  void EmitUnconditionalBranchRegOp(UnconditionalBranchRegOp op, Register rn) {
    ASSERT((rn != CSP) && (rn != R31));
    const int32_t encoding = op | Arm64Encode::Rn(rn);
    Emit(encoding);
  }

  static int32_t ExceptionGenOpEncoding(ExceptionGenOp op, uint16_t imm) {
    return op | (static_cast<int32_t>(imm) << kImm16Shift);
  }

  void EmitExceptionGenOp(ExceptionGenOp op, uint16_t imm) {
    Emit(ExceptionGenOpEncoding(op, imm));
  }

  void EmitMoveWideOp(MoveWideOp op,
                      Register rd,
                      const Immediate& imm,
                      int hw_idx,
                      OperandSize sz) {
    ASSERT((hw_idx >= 0) && (hw_idx <= 3));
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding =
        op | size | Arm64Encode::Rd(rd) |
        (static_cast<int32_t>(hw_idx) << kHWShift) |
        (static_cast<int32_t>(imm.value() & 0xffff) << kImm16Shift);
    Emit(encoding);
  }

  void EmitLoadStoreExclusive(LoadStoreExclusiveOp op,
                              Register rs,
                              Register rn,
                              Register rt,
                              OperandSize sz = kDoubleWord) {
    ASSERT(sz == kDoubleWord || sz == kWord);
    const int32_t size = B31 | (sz == kDoubleWord ? B30 : 0);

    ASSERT((rs != kNoRegister) && (rs != ZR));
    ASSERT((rn != kNoRegister) && (rn != ZR));
    ASSERT((rt != kNoRegister) && (rt != ZR));

    const int32_t encoding = op | size | Arm64Encode::Rs(rs) |
                             Arm64Encode::Rt2(R31) | Arm64Encode::Rn(rn) |
                             Arm64Encode::Rt(rt);

    Emit(encoding);
  }

  void EmitLoadStoreReg(LoadStoreRegOp op,
                        Register rt,
                        Address a,
                        OperandSize sz) {
    // Unpredictable, illegal on some microarchitectures.
    ASSERT((op != LDR && op != STR && op != LDRS) || a.can_writeback_to(rt));

    const int32_t size = Log2OperandSizeBytes(sz);
    const int32_t encoding =
        op | ((size & 0x3) << kSzShift) | Arm64Encode::Rt(rt) | a.encoding();
    Emit(encoding);
  }

  void EmitLoadRegLiteral(LoadRegLiteralOp op,
                          Register rt,
                          Address a,
                          OperandSize sz) {
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    ASSERT((rt != CSP) && (rt != R31));
    const int32_t size = (sz == kDoubleWord) ? B30 : 0;
    const int32_t encoding = op | size | Arm64Encode::Rt(rt) | a.encoding();
    Emit(encoding);
  }

  void EmitLoadStoreRegPair(LoadStoreRegPairOp op,
                            Register rt,
                            Register rt2,
                            Address a,
                            OperandSize sz) {
    // Unpredictable, illegal on some microarchitectures.
    ASSERT(a.can_writeback_to(rt) && a.can_writeback_to(rt2));
    ASSERT(op != LDP || rt != rt2);

    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    ASSERT((rt != CSP) && (rt != R31));
    ASSERT((rt2 != CSP) && (rt2 != R31));
    int32_t opc = 0;
    switch (sz) {
      case kDoubleWord:
        opc = B31;
        break;
      case kWord:
        opc = B30;
        break;
      case kUnsignedWord:
        opc = 0;
        break;
      default:
        UNREACHABLE();
        break;
    }
    const int32_t encoding =
        opc | op | Arm64Encode::Rt(rt) | Arm64Encode::Rt2(rt2) | a.encoding();
    Emit(encoding);
  }

  void EmitPCRelOp(PCRelOp op, Register rd, const Immediate& imm) {
    ASSERT(Utils::IsInt(21, imm.value()));
    ASSERT((rd != R31) && (rd != CSP));
    const int32_t loimm = (imm.value() & 0x3) << 29;
    const int32_t hiimm =
        (static_cast<uint32_t>(imm.value() >> 2) << kImm19Shift) & kImm19Mask;
    const int32_t encoding = op | loimm | hiimm | Arm64Encode::Rd(rd);
    Emit(encoding);
  }

  void EmitMiscDP1Source(MiscDP1SourceOp op,
                         Register rd,
                         Register rn,
                         OperandSize sz) {
    ASSERT((rd != CSP) && (rn != CSP));
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding =
        op | size | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn);
    Emit(encoding);
  }

  void EmitMiscDP2Source(MiscDP2SourceOp op,
                         Register rd,
                         Register rn,
                         Register rm,
                         OperandSize sz) {
    ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP));
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding = op | size | Arm64Encode::Rd(rd) |
                             Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm);
    Emit(encoding);
  }

  void EmitMiscDP3Source(MiscDP3SourceOp op,
                         Register rd,
                         Register rn,
                         Register rm,
                         Register ra,
                         OperandSize sz) {
    ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP) && (ra != CSP));
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding = op | size | Arm64Encode::Rd(rd) |
                             Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm) |
                             Arm64Encode::Ra(ra);
    Emit(encoding);
  }

  void EmitConditionalSelect(ConditionalSelectOp op,
                             Register rd,
                             Register rn,
                             Register rm,
                             Condition cond,
                             OperandSize sz) {
    ASSERT((rd != CSP) && (rn != CSP) && (rm != CSP));
    ASSERT((sz == kDoubleWord) || (sz == kWord) || (sz == kUnsignedWord));
    const int32_t size = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding = op | size | Arm64Encode::Rd(rd) |
                             Arm64Encode::Rn(rn) | Arm64Encode::Rm(rm) |
                             (static_cast<int32_t>(cond) << kSelCondShift);
    Emit(encoding);
  }

  void EmitFPImm(FPImmOp op, VRegister vd, uint8_t imm8) {
    const int32_t encoding =
        op | (static_cast<int32_t>(vd) << kVdShift) | (imm8 << kImm8Shift);
    Emit(encoding);
  }

  void EmitFPIntCvtOp(FPIntCvtOp op,
                      Register rd,
                      Register rn,
                      OperandSize sz = kDoubleWord) {
    ASSERT((sz == kDoubleWord) || (sz == kWord));
    const int32_t sfield = (sz == kDoubleWord) ? B31 : 0;
    const int32_t encoding =
        op | Arm64Encode::Rd(rd) | Arm64Encode::Rn(rn) | sfield;
    Emit(encoding);
  }

  void EmitFPOneSourceOp(FPOneSourceOp op, VRegister vd, VRegister vn) {
    const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                             (static_cast<int32_t>(vn) << kVnShift);
    Emit(encoding);
  }

  void EmitFPTwoSourceOp(FPTwoSourceOp op,
                         VRegister vd,
                         VRegister vn,
                         VRegister vm) {
    const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                             (static_cast<int32_t>(vn) << kVnShift) |
                             (static_cast<int32_t>(vm) << kVmShift);
    Emit(encoding);
  }

  void EmitFPCompareOp(FPCompareOp op, VRegister vn, VRegister vm) {
    const int32_t encoding = op | (static_cast<int32_t>(vn) << kVnShift) |
                             (static_cast<int32_t>(vm) << kVmShift);
    Emit(encoding);
  }

  void EmitSIMDThreeSameOp(SIMDThreeSameOp op,
                           VRegister vd,
                           VRegister vn,
                           VRegister vm) {
    const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                             (static_cast<int32_t>(vn) << kVnShift) |
                             (static_cast<int32_t>(vm) << kVmShift);
    Emit(encoding);
  }

  void EmitSIMDCopyOp(SIMDCopyOp op,
                      VRegister vd,
                      VRegister vn,
                      OperandSize sz,
                      int32_t idx4,
                      int32_t idx5) {
    const int32_t shift = Log2OperandSizeBytes(sz);
    const int32_t imm5 = ((idx5 << (shift + 1)) | (1 << shift)) & 0x1f;
    const int32_t imm4 = (idx4 << shift) & 0xf;
    const int32_t encoding = op | (imm5 << kImm5Shift) | (imm4 << kImm4Shift) |
                             (static_cast<int32_t>(vd) << kVdShift) |
                             (static_cast<int32_t>(vn) << kVnShift);
    Emit(encoding);
  }

  void EmitSIMDTwoRegOp(SIMDTwoRegOp op, VRegister vd, VRegister vn) {
    const int32_t encoding = op | (static_cast<int32_t>(vd) << kVdShift) |
                             (static_cast<int32_t>(vn) << kVnShift);
    Emit(encoding);
  }

  enum BarrierFilterMode {
    // Filter falls through into the barrier update code. Target label
    // is a "after-store" label.
    kJumpToNoUpdate,

    // Filter falls through to the "after-store" code. Target label
    // is barrier update code label.
    kJumpToBarrier,
  };

  void StoreIntoObjectFilter(Register object,
                             Register value,
                             Label* label,
                             CanBeSmi can_be_smi,
                             BarrierFilterMode barrier_filter_mode);

  friend class dart::FlowGraphCompiler;
  std::function<void(Register reg)> generate_invoke_write_barrier_wrapper_;
  std::function<void()> generate_invoke_array_write_barrier_;

  DISALLOW_ALLOCATION();
  DISALLOW_COPY_AND_ASSIGN(Assembler);
};

}  // namespace compiler
}  // namespace dart

#endif  // RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_ARM64_H_