// Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

#include "vm/globals.h"  // NOLINT
#if defined(TARGET_ARCH_X64)

#define SHOULD_NOT_INCLUDE_RUNTIME

#include "vm/class_id.h"
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/instructions.h"
#include "vm/tags.h"

namespace dart {

DECLARE_FLAG(bool, check_code_pointer);
DECLARE_FLAG(bool, precompiled_mode);

namespace compiler {

Assembler::Assembler(ObjectPoolBuilder* object_pool_builder,
                     intptr_t far_branch_level)
    : AssemblerBase(object_pool_builder), constant_pool_allowed_(false) {
  // Far branching mode is only needed and implemented for ARM.
  ASSERT(far_branch_level == 0);

  generate_invoke_write_barrier_wrapper_ = [&](Register reg) {
    call(Address(THR,
                 target::Thread::write_barrier_wrappers_thread_offset(reg)));
  };
  generate_invoke_array_write_barrier_ = [&]() {
    call(
        Address(THR, target::Thread::array_write_barrier_entry_point_offset()));
  };
}

void Assembler::call(Label* label) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  static const int kSize = 5;
  EmitUint8(0xE8);
  EmitLabel(label, kSize);
}

void Assembler::LoadNativeEntry(
    Register dst,
    const ExternalLabel* label,
    ObjectPoolBuilderEntry::Patchability patchable) {
  const intptr_t index =
      object_pool_builder().FindNativeFunction(label, patchable);
  LoadWordFromPoolIndex(dst, index);
}

void Assembler::call(const ExternalLabel* label) {
  {  // Encode movq(TMP, Immediate(label->address())), but always as imm64.
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitRegisterREX(TMP, REX_W);
    EmitUint8(0xB8 | (TMP & 7));
    EmitInt64(label->address());
  }
  call(TMP);
}

void Assembler::CallPatchable(const Code& target, CodeEntryKind entry_kind) {
  ASSERT(constant_pool_allowed());
  const intptr_t idx = object_pool_builder().AddObject(
      ToObject(target), ObjectPoolBuilderEntry::kPatchable);
  LoadWordFromPoolIndex(CODE_REG, idx);
  call(FieldAddress(CODE_REG, target::Code::entry_point_offset(entry_kind)));
}

void Assembler::CallWithEquivalence(const Code& target,
                                    const Object& equivalence,
                                    CodeEntryKind entry_kind) {
  ASSERT(constant_pool_allowed());
  const intptr_t idx =
      object_pool_builder().FindObject(ToObject(target), equivalence);
  LoadWordFromPoolIndex(CODE_REG, idx);
  call(FieldAddress(CODE_REG, target::Code::entry_point_offset(entry_kind)));
}

void Assembler::Call(const Code& target) {
  ASSERT(constant_pool_allowed());
  const intptr_t idx = object_pool_builder().FindObject(
      ToObject(target), ObjectPoolBuilderEntry::kNotPatchable);
  LoadWordFromPoolIndex(CODE_REG, idx);
  call(FieldAddress(CODE_REG, target::Code::entry_point_offset()));
}

void Assembler::pushq(Register reg) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitRegisterREX(reg, REX_NONE);
  EmitUint8(0x50 | (reg & 7));
}

void Assembler::pushq(const Immediate& imm) {
  if (imm.is_int8()) {
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitUint8(0x6A);
    EmitUint8(imm.value() & 0xFF);
  } else if (imm.is_int32()) {
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitUint8(0x68);
    EmitImmediate(imm);
  } else {
    movq(TMP, imm);
    pushq(TMP);
  }
}

void Assembler::popq(Register reg) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitRegisterREX(reg, REX_NONE);
  EmitUint8(0x58 | (reg & 7));
}

void Assembler::setcc(Condition condition, ByteRegister dst) {
  ASSERT(dst != kNoByteRegister);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (dst >= 8) {
    EmitUint8(REX_PREFIX | (((dst & 0x08) != 0) ? REX_B : REX_NONE));
  }
  EmitUint8(0x0F);
  EmitUint8(0x90 + condition);
  EmitUint8(0xC0 + (dst & 0x07));
}

void Assembler::EnterFullSafepoint() {
  // We generate the same number of instructions whether or not the slow-path is
  // forced, to simplify GenerateJitCallbackTrampolines.
  Label done, slow_path;
  if (FLAG_use_slow_path) {
    jmp(&slow_path);
  }

  // Compare and swap the value at Thread::safepoint_state from
  // unacquired to acquired. If the CAS fails, go to a slow-path stub.
  pushq(RAX);
  movq(RAX, Immediate(target::Thread::full_safepoint_state_unacquired()));
  movq(TMP, Immediate(target::Thread::full_safepoint_state_acquired()));
  LockCmpxchgq(Address(THR, target::Thread::safepoint_state_offset()), TMP);
  movq(TMP, RAX);
  popq(RAX);
  cmpq(TMP, Immediate(target::Thread::full_safepoint_state_unacquired()));

  if (!FLAG_use_slow_path) {
    j(EQUAL, &done);
  }

  Bind(&slow_path);
  movq(TMP, Address(THR, target::Thread::enter_safepoint_stub_offset()));
  movq(TMP, FieldAddress(TMP, target::Code::entry_point_offset()));

  // Use call instead of CallCFunction to avoid having to clean up shadow space
  // afterwards. This is possible because the safepoint stub does not use the
  // shadow space as scratch and has no arguments.
  call(TMP);

  Bind(&done);
}

void Assembler::TransitionGeneratedToNative(Register destination_address,
                                            Register new_exit_frame,
                                            Register new_exit_through_ffi,
                                            bool enter_safepoint) {
  // Save exit frame information to enable stack walking.
  movq(Address(THR, target::Thread::top_exit_frame_info_offset()),
       new_exit_frame);

  movq(compiler::Address(THR,
                         compiler::target::Thread::exit_through_ffi_offset()),
       new_exit_through_ffi);

  movq(Assembler::VMTagAddress(), destination_address);
  movq(Address(THR, target::Thread::execution_state_offset()),
       Immediate(target::Thread::native_execution_state()));

  if (enter_safepoint) {
    EnterFullSafepoint();
  }
}

void Assembler::ExitFullSafepoint(bool ignore_unwind_in_progress) {
  // We generate the same number of instructions whether or not the slow-path is
  // forced, for consistency with EnterFullSafepoint.
  Label done, slow_path;
  if (FLAG_use_slow_path) {
    jmp(&slow_path);
  }

  // Compare and swap the value at Thread::safepoint_state from
  // acquired to unacquired. On success, jump to 'success'; otherwise,
  // fallthrough.

  pushq(RAX);
  movq(RAX, Immediate(target::Thread::full_safepoint_state_acquired()));
  movq(TMP, Immediate(target::Thread::full_safepoint_state_unacquired()));
  LockCmpxchgq(Address(THR, target::Thread::safepoint_state_offset()), TMP);
  movq(TMP, RAX);
  popq(RAX);
  cmpq(TMP, Immediate(target::Thread::full_safepoint_state_acquired()));

  if (!FLAG_use_slow_path) {
    j(EQUAL, &done);
  }

  Bind(&slow_path);
  if (ignore_unwind_in_progress) {
    movq(TMP,
         Address(THR,
                 target::Thread::
                     exit_safepoint_ignore_unwind_in_progress_stub_offset()));
  } else {
    movq(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset()));
  }
  movq(TMP, FieldAddress(TMP, target::Code::entry_point_offset()));

  // Use call instead of CallCFunction to avoid having to clean up shadow space
  // afterwards. This is possible because the safepoint stub does not use the
  // shadow space as scratch and has no arguments.
  call(TMP);

  Bind(&done);
}

void Assembler::TransitionNativeToGenerated(bool leave_safepoint,
                                            bool ignore_unwind_in_progress) {
  if (leave_safepoint) {
    ExitFullSafepoint(ignore_unwind_in_progress);
  } else {
    // flag only makes sense if we are leaving safepoint
    ASSERT(!ignore_unwind_in_progress);
#if defined(DEBUG)
    // Ensure we've already left the safepoint.
    movq(TMP, Address(THR, target::Thread::safepoint_state_offset()));
    andq(TMP, Immediate(target::Thread::full_safepoint_state_acquired()));
    Label ok;
    j(ZERO, &ok);
    Breakpoint();
    Bind(&ok);
#endif
  }

  movq(Assembler::VMTagAddress(), Immediate(target::Thread::vm_tag_dart_id()));
  movq(Address(THR, target::Thread::execution_state_offset()),
       Immediate(target::Thread::generated_execution_state()));

  // Reset exit frame information in Isolate's mutator thread structure.
  movq(Address(THR, target::Thread::top_exit_frame_info_offset()),
       Immediate(0));
  movq(compiler::Address(THR,
                         compiler::target::Thread::exit_through_ffi_offset()),
       compiler::Immediate(0));
}

void Assembler::EmitQ(int reg,
                      const Address& address,
                      int opcode,
                      int prefix2,
                      int prefix1) {
  ASSERT(reg <= XMM15);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (prefix1 >= 0) {
    EmitUint8(prefix1);
  }
  EmitOperandREX(reg, address, REX_W);
  if (prefix2 >= 0) {
    EmitUint8(prefix2);
  }
  EmitUint8(opcode);
  EmitOperand(reg & 7, address);
}

void Assembler::EmitL(int reg,
                      const Address& address,
                      int opcode,
                      int prefix2,
                      int prefix1) {
  ASSERT(reg <= XMM15);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (prefix1 >= 0) {
    EmitUint8(prefix1);
  }
  EmitOperandREX(reg, address, REX_NONE);
  if (prefix2 >= 0) {
    EmitUint8(prefix2);
  }
  EmitUint8(opcode);
  EmitOperand(reg & 7, address);
}

void Assembler::EmitW(Register reg,
                      const Address& address,
                      int opcode,
                      int prefix2,
                      int prefix1) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (prefix1 >= 0) {
    EmitUint8(prefix1);
  }
  EmitOperandSizeOverride();
  EmitOperandREX(reg, address, REX_NONE);
  if (prefix2 >= 0) {
    EmitUint8(prefix2);
  }
  EmitUint8(opcode);
  EmitOperand(reg & 7, address);
}

void Assembler::EmitB(int reg, const Address& address, int opcode) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandREX(reg & ~0x10, address, reg >= 8 ? REX_PREFIX : REX_NONE);
  EmitUint8(opcode);
  EmitOperand(reg & 7, address);
}

void Assembler::movl(Register dst, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  Operand operand(dst);
  EmitOperandREX(0, operand, REX_NONE);
  EmitUint8(0xC7);
  EmitOperand(0, operand);
  ASSERT(imm.is_int32());
  EmitImmediate(imm);
}

void Assembler::movl(const Address& dst, const Immediate& imm) {
  movl(TMP, imm);
  movl(dst, TMP);
}

void Assembler::movb(const Address& dst, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandREX(0, dst, REX_NONE);
  EmitUint8(0xC6);
  EmitOperand(0, dst);
  ASSERT(imm.is_int8());
  EmitUint8(imm.value() & 0xFF);
}

void Assembler::movw(Register dst, const Address& src) {
  // This would leave 16 bits above the 2 byte value undefined.
  // If we ever want to purposefully have those undefined, remove this.
  // TODO(40210): Allow this.
  FATAL("Use movzxw or movsxw instead.");
}

void Assembler::movw(const Address& dst, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandSizeOverride();
  EmitOperandREX(0, dst, REX_NONE);
  EmitUint8(0xC7);
  EmitOperand(0, dst);
  EmitUint8(imm.value() & 0xFF);
  EmitUint8((imm.value() >> 8) & 0xFF);
}

void Assembler::movq(Register dst, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (imm.is_uint32()) {
    // Pick single byte B8 encoding if possible. If dst < 8 then we also omit
    // the Rex byte.
    EmitRegisterREX(dst, REX_NONE);
    EmitUint8(0xB8 | (dst & 7));
    EmitUInt32(imm.value());
  } else if (imm.is_int32()) {
    // Sign extended C7 Cx encoding if we have a negative input.
    Operand operand(dst);
    EmitOperandREX(0, operand, REX_W);
    EmitUint8(0xC7);
    EmitOperand(0, operand);
    EmitImmediate(imm);
  } else {
    // Full 64 bit immediate encoding.
    EmitRegisterREX(dst, REX_W);
    EmitUint8(0xB8 | (dst & 7));
    EmitImmediate(imm);
  }
}

void Assembler::movq(const Address& dst, const Immediate& imm) {
  if (imm.is_int32()) {
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitOperandREX(0, dst, REX_W);
    EmitUint8(0xC7);
    EmitOperand(0, dst);
    EmitImmediate(imm);
  } else {
    movq(TMP, imm);
    movq(dst, TMP);
  }
}

void Assembler::EmitSimple(int opcode, int opcode2, int opcode3) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitUint8(opcode);
  if (opcode2 != -1) {
    EmitUint8(opcode2);
    if (opcode3 != -1) {
      EmitUint8(opcode3);
    }
  }
}

void Assembler::EmitQ(int dst, int src, int opcode, int prefix2, int prefix1) {
  ASSERT(src <= XMM15);
  ASSERT(dst <= XMM15);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (prefix1 >= 0) {
    EmitUint8(prefix1);
  }
  EmitRegRegRex(dst, src, REX_W);
  if (prefix2 >= 0) {
    EmitUint8(prefix2);
  }
  EmitUint8(opcode);
  EmitRegisterOperand(dst & 7, src);
}

void Assembler::EmitL(int dst, int src, int opcode, int prefix2, int prefix1) {
  ASSERT(src <= XMM15);
  ASSERT(dst <= XMM15);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (prefix1 >= 0) {
    EmitUint8(prefix1);
  }
  EmitRegRegRex(dst, src);
  if (prefix2 >= 0) {
    EmitUint8(prefix2);
  }
  EmitUint8(opcode);
  EmitRegisterOperand(dst & 7, src);
}

void Assembler::EmitW(Register dst,
                      Register src,
                      int opcode,
                      int prefix2,
                      int prefix1) {
  ASSERT(src <= R15);
  ASSERT(dst <= R15);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (prefix1 >= 0) {
    EmitUint8(prefix1);
  }
  EmitOperandSizeOverride();
  EmitRegRegRex(dst, src);
  if (prefix2 >= 0) {
    EmitUint8(prefix2);
  }
  EmitUint8(opcode);
  EmitRegisterOperand(dst & 7, src);
}

#define UNARY_XMM_WITH_CONSTANT(name, constant, op)                            \
  void Assembler::name(XmmRegister dst, XmmRegister src) {                     \
    movq(TMP, Address(THR, target::Thread::constant##_address_offset()));      \
    if (dst == src) {                                                          \
      op(dst, Address(TMP, 0));                                                \
    } else {                                                                   \
      movups(dst, Address(TMP, 0));                                            \
      op(dst, src);                                                            \
    }                                                                          \
  }

// TODO(erikcorry): For the case where dst != src, we could construct these
// with pcmpeqw xmm0,xmm0 followed by left and right shifts.  This would avoid
// memory traffic.
// { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
UNARY_XMM_WITH_CONSTANT(notps, float_not, xorps)
// { 0x80000000, 0x80000000, 0x80000000, 0x80000000 }
UNARY_XMM_WITH_CONSTANT(negateps, float_negate, xorps)
// { 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF }
UNARY_XMM_WITH_CONSTANT(absps, float_absolute, andps)
// { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 }
UNARY_XMM_WITH_CONSTANT(zerowps, float_zerow, andps)
// { 0x8000000000000000LL, 0x8000000000000000LL }
UNARY_XMM_WITH_CONSTANT(negatepd, double_negate, xorpd)
// { 0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL }
UNARY_XMM_WITH_CONSTANT(abspd, double_abs, andpd)
// {0x8000000000000000LL, 0x8000000000000000LL}
UNARY_XMM_WITH_CONSTANT(DoubleNegate, double_negate, xorpd)
// {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL}
UNARY_XMM_WITH_CONSTANT(DoubleAbs, double_abs, andpd)

#undef UNARY_XMM_WITH_CONSTANT

void Assembler::CmpPS(XmmRegister dst, XmmRegister src, int condition) {
  EmitL(dst, src, 0xC2, 0x0F);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitUint8(condition);
}

void Assembler::set1ps(XmmRegister dst, Register tmp1, const Immediate& imm) {
  // Load 32-bit immediate value into tmp1.
  movl(tmp1, imm);
  // Move value from tmp1 into dst.
  movd(dst, tmp1);
  // Broadcast low lane into other three lanes.
  shufps(dst, dst, Immediate(0x0));
}

void Assembler::shufps(XmmRegister dst, XmmRegister src, const Immediate& imm) {
  EmitL(dst, src, 0xC6, 0x0F);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  ASSERT(imm.is_uint8());
  EmitUint8(imm.value());
}

void Assembler::shufpd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
  EmitL(dst, src, 0xC6, 0x0F, 0x66);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  ASSERT(imm.is_uint8());
  EmitUint8(imm.value());
}

void Assembler::roundsd(XmmRegister dst, XmmRegister src, RoundingMode mode) {
  ASSERT(src <= XMM15);
  ASSERT(dst <= XMM15);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitUint8(0x66);
  EmitRegRegRex(dst, src);
  EmitUint8(0x0F);
  EmitUint8(0x3A);
  EmitUint8(0x0B);
  EmitRegisterOperand(dst & 7, src);
  // Mask precision exception.
  EmitUint8(static_cast<uint8_t>(mode) | 0x8);
}

void Assembler::fldl(const Address& src) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitUint8(0xDD);
  EmitOperand(0, src);
}

void Assembler::fstpl(const Address& dst) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitUint8(0xDD);
  EmitOperand(3, dst);
}

void Assembler::ffree(intptr_t value) {
  ASSERT(value < 7);
  EmitSimple(0xDD, 0xC0 + value);
}

void Assembler::CompareImmediate(Register reg,
                                 const Immediate& imm,
                                 OperandSize width) {
  if (width == kEightBytes) {
    if (imm.is_int32()) {
      cmpq(reg, imm);
    } else {
      ASSERT(reg != TMP);
      LoadImmediate(TMP, imm);
      cmpq(reg, TMP);
    }
  } else {
    ASSERT(width == kFourBytes);
    cmpl(reg, imm);
  }
}

void Assembler::CompareImmediate(const Address& address,
                                 const Immediate& imm,
                                 OperandSize width) {
  if (width == kEightBytes) {
    if (imm.is_int32()) {
      cmpq(address, imm);
    } else {
      LoadImmediate(TMP, imm);
      cmpq(address, TMP);
    }
  } else {
    ASSERT(width == kFourBytes);
    cmpl(address, imm);
  }
}

void Assembler::testb(const Address& address, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandREX(0, address, REX_NONE);
  EmitUint8(0xF6);
  EmitOperand(0, address);
  ASSERT(imm.is_int8());
  EmitUint8(imm.value() & 0xFF);
}

void Assembler::testb(const Address& address, Register reg) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandREX(reg, address, REX_NONE);
  EmitUint8(0x84);
  EmitOperand(reg & 7, address);
}

void Assembler::testq(Register reg, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (imm.is_uint8()) {
    // Use zero-extended 8-bit immediate.
    if (reg >= 4) {
      // We need the Rex byte to give access to the SIL and DIL registers (the
      // low bytes of RSI and RDI).
      EmitRegisterREX(reg, REX_NONE, /* force = */ true);
    }
    if (reg == RAX) {
      EmitUint8(0xA8);
    } else {
      EmitUint8(0xF6);
      EmitUint8(0xC0 + (reg & 7));
    }
    EmitUint8(imm.value() & 0xFF);
  } else if (imm.is_uint32()) {
    if (reg == RAX) {
      EmitUint8(0xA9);
    } else {
      EmitRegisterREX(reg, REX_NONE);
      EmitUint8(0xF7);
      EmitUint8(0xC0 | (reg & 7));
    }
    EmitUInt32(imm.value());
  } else {
    // Sign extended version of 32 bit test.
    ASSERT(imm.is_int32());
    EmitRegisterREX(reg, REX_W);
    if (reg == RAX) {
      EmitUint8(0xA9);
    } else {
      EmitUint8(0xF7);
      EmitUint8(0xC0 | (reg & 7));
    }
    EmitImmediate(imm);
  }
}

void Assembler::TestImmediate(Register dst,
                              const Immediate& imm,
                              OperandSize width) {
  if (width == kEightBytes) {
    if (imm.is_int32() || imm.is_uint32()) {
      testq(dst, imm);
    } else {
      ASSERT(dst != TMP);
      LoadImmediate(TMP, imm);
      testq(dst, TMP);
    }
  } else {
    ASSERT(width == kFourBytes);
    testl(dst, imm);
  }
}

void Assembler::AluL(uint8_t modrm_opcode, Register dst, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitRegisterREX(dst, REX_NONE);
  EmitComplex(modrm_opcode, Operand(dst), imm);
}

void Assembler::AluB(uint8_t modrm_opcode,
                     const Address& dst,
                     const Immediate& imm) {
  ASSERT(imm.is_uint8() || imm.is_int8());
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandREX(modrm_opcode, dst, REX_NONE);
  EmitUint8(0x80);
  EmitOperand(modrm_opcode, dst);
  EmitUint8(imm.value() & 0xFF);
}

void Assembler::AluW(uint8_t modrm_opcode,
                     const Address& dst,
                     const Immediate& imm) {
  ASSERT(imm.is_int16() || imm.is_uint16());
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandSizeOverride();
  EmitOperandREX(modrm_opcode, dst, REX_NONE);
  if (imm.is_int8()) {
    EmitSignExtendedInt8(modrm_opcode, dst, imm);
  } else {
    EmitUint8(0x81);
    EmitOperand(modrm_opcode, dst);
    EmitUint8(imm.value() & 0xFF);
    EmitUint8((imm.value() >> 8) & 0xFF);
  }
}

void Assembler::AluL(uint8_t modrm_opcode,
                     const Address& dst,
                     const Immediate& imm) {
  ASSERT(imm.is_int32());
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitOperandREX(modrm_opcode, dst, REX_NONE);
  EmitComplex(modrm_opcode, dst, imm);
}

void Assembler::AluQ(uint8_t modrm_opcode,
                     uint8_t opcode,
                     Register dst,
                     const Immediate& imm) {
  Operand operand(dst);
  if (modrm_opcode == 4 && imm.is_uint32()) {
    // We can use andl for andq.
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitRegisterREX(dst, REX_NONE);
    // Would like to use EmitComplex here, but it doesn't like uint32
    // immediates.
    if (imm.is_int8()) {
      EmitSignExtendedInt8(modrm_opcode, operand, imm);
    } else {
      if (dst == RAX) {
        EmitUint8(0x25);
      } else {
        EmitUint8(0x81);
        EmitOperand(modrm_opcode, operand);
      }
      EmitUInt32(imm.value());
    }
  } else if (imm.is_int32()) {
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitRegisterREX(dst, REX_W);
    EmitComplex(modrm_opcode, operand, imm);
  } else {
    ASSERT(dst != TMP);
    movq(TMP, imm);
    EmitQ(dst, TMP, opcode);
  }
}

void Assembler::AluQ(uint8_t modrm_opcode,
                     uint8_t opcode,
                     const Address& dst,
                     const Immediate& imm) {
  if (imm.is_int32()) {
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitOperandREX(modrm_opcode, dst, REX_W);
    EmitComplex(modrm_opcode, dst, imm);
  } else {
    movq(TMP, imm);
    EmitQ(TMP, dst, opcode);
  }
}

void Assembler::AndImmediate(Register dst, const Immediate& imm) {
  if (imm.is_int32() || imm.is_uint32()) {
    andq(dst, imm);
  } else {
    ASSERT(dst != TMP);
    LoadImmediate(TMP, imm);
    andq(dst, TMP);
  }
}

void Assembler::AndRegisters(Register dst, Register src1, Register src2) {
  ASSERT(src1 != src2);  // Likely a mistake.
  if (src2 == kNoRegister) {
    src2 = dst;
  }
  if (dst == src1) {
    andq(dst, src2);
  } else if (dst == src2) {
    andq(dst, src1);
  } else {
    movq(dst, src1);
    andq(dst, src2);
  }
}

void Assembler::LslRegister(Register dst, Register shift) {
  if (shift != RCX) {
    movq(TMP, RCX);
    movq(RCX, shift);
    shlq(dst == RCX ? TMP : dst, RCX);
    movq(RCX, TMP);
  } else {
    shlq(dst, shift);
  }
}

void Assembler::OrImmediate(Register dst, const Immediate& imm) {
  if (imm.is_int32()) {
    orq(dst, imm);
  } else {
    ASSERT(dst != TMP);
    LoadImmediate(TMP, imm);
    orq(dst, TMP);
  }
}

void Assembler::XorImmediate(Register dst, const Immediate& imm) {
  if (imm.is_int32()) {
    xorq(dst, imm);
  } else {
    ASSERT(dst != TMP);
    LoadImmediate(TMP, imm);
    xorq(dst, TMP);
  }
}

void Assembler::cqo() {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitRegisterREX(RAX, REX_W);
  EmitUint8(0x99);
}

void Assembler::EmitUnaryQ(Register reg, int opcode, int modrm_code) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitRegisterREX(reg, REX_W);
  EmitUint8(opcode);
  EmitOperand(modrm_code, Operand(reg));
}

void Assembler::EmitUnaryL(Register reg, int opcode, int modrm_code) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitRegisterREX(reg, REX_NONE);
  EmitUint8(opcode);
  EmitOperand(modrm_code, Operand(reg));
}

void Assembler::EmitUnaryQ(const Address& address, int opcode, int modrm_code) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  Operand operand(address);
  EmitOperandREX(modrm_code, operand, REX_W);
  EmitUint8(opcode);
  EmitOperand(modrm_code, operand);
}

void Assembler::EmitUnaryL(const Address& address, int opcode, int modrm_code) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  Operand operand(address);
  EmitOperandREX(modrm_code, operand, REX_NONE);
  EmitUint8(opcode);
  EmitOperand(modrm_code, operand);
}

void Assembler::imull(Register reg, const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  Operand operand(reg);
  EmitOperandREX(reg, operand, REX_NONE);
  EmitUint8(0x69);
  EmitOperand(reg & 7, Operand(reg));
  EmitImmediate(imm);
}

void Assembler::imulq(Register reg, const Immediate& imm) {
  if (imm.is_int32()) {
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    Operand operand(reg);
    EmitOperandREX(reg, operand, REX_W);
    EmitUint8(0x69);
    EmitOperand(reg & 7, Operand(reg));
    EmitImmediate(imm);
  } else {
    ASSERT(reg != TMP);
    movq(TMP, imm);
    imulq(reg, TMP);
  }
}

void Assembler::MulImmediate(Register reg,
                             const Immediate& imm,
                             OperandSize width) {
  ASSERT(width == kFourBytes || width == kEightBytes);
  if (Utils::IsPowerOfTwo(imm.value())) {
    const intptr_t shift = Utils::ShiftForPowerOfTwo(imm.value());
    if (width == kFourBytes) {
      shll(reg, Immediate(shift));
    } else {
      shlq(reg, Immediate(shift));
    }
  } else if (imm.is_int32()) {
    if (width == kFourBytes) {
      imull(reg, imm);
    } else {
      imulq(reg, imm);
    }
  } else {
    ASSERT(reg != TMP);
    ASSERT(width == kEightBytes);
    movq(TMP, imm);
    imulq(reg, TMP);
  }
}

void Assembler::shll(Register reg, const Immediate& imm) {
  EmitGenericShift(false, 4, reg, imm);
}

void Assembler::shll(Register operand, Register shifter) {
  EmitGenericShift(false, 4, operand, shifter);
}

void Assembler::shrl(Register reg, const Immediate& imm) {
  EmitGenericShift(false, 5, reg, imm);
}

void Assembler::shrl(Register operand, Register shifter) {
  EmitGenericShift(false, 5, operand, shifter);
}

void Assembler::sarl(Register reg, const Immediate& imm) {
  EmitGenericShift(false, 7, reg, imm);
}

void Assembler::sarl(Register operand, Register shifter) {
  EmitGenericShift(false, 7, operand, shifter);
}

void Assembler::shldl(Register dst, Register src, const Immediate& imm) {
  EmitL(src, dst, 0xA4, 0x0F);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  ASSERT(imm.is_int8());
  EmitUint8(imm.value() & 0xFF);
}

void Assembler::shlq(Register reg, const Immediate& imm) {
  EmitGenericShift(true, 4, reg, imm);
}

void Assembler::shlq(Register operand, Register shifter) {
  EmitGenericShift(true, 4, operand, shifter);
}

void Assembler::shrq(Register reg, const Immediate& imm) {
  EmitGenericShift(true, 5, reg, imm);
}

void Assembler::shrq(Register operand, Register shifter) {
  EmitGenericShift(true, 5, operand, shifter);
}

void Assembler::sarq(Register reg, const Immediate& imm) {
  EmitGenericShift(true, 7, reg, imm);
}

void Assembler::sarq(Register operand, Register shifter) {
  EmitGenericShift(true, 7, operand, shifter);
}

void Assembler::shldq(Register dst, Register src, const Immediate& imm) {
  EmitQ(src, dst, 0xA4, 0x0F);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  ASSERT(imm.is_int8());
  EmitUint8(imm.value() & 0xFF);
}

void Assembler::btq(Register base, int bit) {
  ASSERT(bit >= 0 && bit < 64);
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  Operand operand(base);
  EmitOperandREX(4, operand, bit >= 32 ? REX_W : REX_NONE);
  EmitUint8(0x0F);
  EmitUint8(0xBA);
  EmitOperand(4, operand);
  EmitUint8(bit);
}

void Assembler::enter(const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  EmitUint8(0xC8);
  ASSERT(imm.is_uint16());
  EmitUint8(imm.value() & 0xFF);
  EmitUint8((imm.value() >> 8) & 0xFF);
  EmitUint8(0x00);
}

void Assembler::nop(int size) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  // There are nops up to size 15, but for now just provide up to size 8.
  ASSERT(0 < size && size <= MAX_NOP_SIZE);
  switch (size) {
    case 1:
      EmitUint8(0x90);
      break;
    case 2:
      EmitUint8(0x66);
      EmitUint8(0x90);
      break;
    case 3:
      EmitUint8(0x0F);
      EmitUint8(0x1F);
      EmitUint8(0x00);
      break;
    case 4:
      EmitUint8(0x0F);
      EmitUint8(0x1F);
      EmitUint8(0x40);
      EmitUint8(0x00);
      break;
    case 5:
      EmitUint8(0x0F);
      EmitUint8(0x1F);
      EmitUint8(0x44);
      EmitUint8(0x00);
      EmitUint8(0x00);
      break;
    case 6:
      EmitUint8(0x66);
      EmitUint8(0x0F);
      EmitUint8(0x1F);
      EmitUint8(0x44);
      EmitUint8(0x00);
      EmitUint8(0x00);
      break;
    case 7:
      EmitUint8(0x0F);
      EmitUint8(0x1F);
      EmitUint8(0x80);
      EmitUint8(0x00);
      EmitUint8(0x00);
      EmitUint8(0x00);
      EmitUint8(0x00);
      break;
    case 8:
      EmitUint8(0x0F);
      EmitUint8(0x1F);
      EmitUint8(0x84);
      EmitUint8(0x00);
      EmitUint8(0x00);
      EmitUint8(0x00);
      EmitUint8(0x00);
      EmitUint8(0x00);
      break;
    default:
      UNIMPLEMENTED();
  }
}

void Assembler::j(Condition condition, Label* label, JumpDistance distance) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (label->IsBound()) {
    static const int kShortSize = 2;
    static const int kLongSize = 6;
    intptr_t offset = label->Position() - buffer_.Size();
    ASSERT(offset <= 0);
    if (Utils::IsInt(8, offset - kShortSize)) {
      EmitUint8(0x70 + condition);
      EmitUint8((offset - kShortSize) & 0xFF);
    } else {
      EmitUint8(0x0F);
      EmitUint8(0x80 + condition);
      EmitInt32(offset - kLongSize);
    }
  } else if (distance == kNearJump) {
    EmitUint8(0x70 + condition);
    EmitNearLabelLink(label);
  } else {
    EmitUint8(0x0F);
    EmitUint8(0x80 + condition);
    EmitLabelLink(label);
  }
}

void Assembler::J(Condition condition, const Code& target, Register pp) {
  Label no_jump;
  // Negate condition.
  j(static_cast<Condition>(condition ^ 1), &no_jump, kNearJump);
  Jmp(target, pp);
  Bind(&no_jump);
}

void Assembler::jmp(Label* label, JumpDistance distance) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  if (label->IsBound()) {
    static const int kShortSize = 2;
    static const int kLongSize = 5;
    intptr_t offset = label->Position() - buffer_.Size();
    ASSERT(offset <= 0);
    if (Utils::IsInt(8, offset - kShortSize)) {
      EmitUint8(0xEB);
      EmitUint8((offset - kShortSize) & 0xFF);
    } else {
      EmitUint8(0xE9);
      EmitInt32(offset - kLongSize);
    }
  } else if (distance == kNearJump) {
    EmitUint8(0xEB);
    EmitNearLabelLink(label);
  } else {
    EmitUint8(0xE9);
    EmitLabelLink(label);
  }
}

void Assembler::jmp(const ExternalLabel* label) {
  {  // Encode movq(TMP, Immediate(label->address())), but always as imm64.
    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    EmitRegisterREX(TMP, REX_W);
    EmitUint8(0xB8 | (TMP & 7));
    EmitInt64(label->address());
  }
  jmp(TMP);
}

void Assembler::JmpPatchable(const Code& target, Register pp) {
  ASSERT((pp != PP) || constant_pool_allowed());
  const intptr_t idx = object_pool_builder().AddObject(
      ToObject(target), ObjectPoolBuilderEntry::kPatchable);
  const int32_t offset = target::ObjectPool::element_offset(idx);
  movq(CODE_REG, Address(pp, offset - kHeapObjectTag));
  movq(TMP, FieldAddress(CODE_REG, target::Code::entry_point_offset()));
  jmp(TMP);
}

void Assembler::Jmp(const Code& target, Register pp) {
  ASSERT((pp != PP) || constant_pool_allowed());
  const intptr_t idx = object_pool_builder().FindObject(
      ToObject(target), ObjectPoolBuilderEntry::kNotPatchable);
  const int32_t offset = target::ObjectPool::element_offset(idx);
  movq(CODE_REG, FieldAddress(pp, offset));
  jmp(FieldAddress(CODE_REG, target::Code::entry_point_offset()));
}

void Assembler::CompareRegisters(Register a, Register b) {
  cmpq(a, b);
}

void Assembler::LoadFromStack(Register dst, intptr_t depth) {
  ASSERT(depth >= 0);
  movq(dst, Address(SPREG, depth * target::kWordSize));
}

void Assembler::StoreToStack(Register src, intptr_t depth) {
  ASSERT(depth >= 0);
  movq(Address(SPREG, depth * target::kWordSize), src);
}

void Assembler::CompareToStack(Register src, intptr_t depth) {
  cmpq(Address(SPREG, depth * target::kWordSize), src);
}

void Assembler::ExtendValue(Register to, Register from, OperandSize sz) {
  switch (sz) {
    case kEightBytes:
      if (to == from) return;  // No operation needed.
      return movq(to, from);
    case kUnsignedFourBytes:
      return movl(to, from);
    case kFourBytes:
      return movsxd(to, from);
    case kUnsignedTwoBytes:
      return movzxw(to, from);
    case kTwoBytes:
      return movsxw(to, from);
    case kUnsignedByte:
      return movzxb(to, from);
    case kByte:
      return movsxb(to, from);
    default:
      UNIMPLEMENTED();
      break;
  }
}

void Assembler::PushRegister(Register r) {
  pushq(r);
}

void Assembler::PopRegister(Register r) {
  popq(r);
}

void Assembler::AddImmediate(Register reg,
                             const Immediate& imm,
                             OperandSize width) {
  ASSERT(width == kFourBytes || width == kEightBytes);
  const int64_t value = imm.value();
  if (value == 0) {
    return;
  }
  if ((value > 0) || (value == kMinInt64)) {
    if (value == 1) {
      if (width == kFourBytes) {
        incl(reg);
      } else {
        incq(reg);
      }
    } else {
      if (imm.is_int32() || (width == kFourBytes && imm.is_uint32())) {
        if (width == kFourBytes) {
          addl(reg, imm);
        } else {
          addq(reg, imm);
        }
      } else {
        ASSERT(reg != TMP);
        ASSERT(width == kEightBytes);
        LoadImmediate(TMP, imm);
        addq(reg, TMP);
      }
    }
  } else {
    SubImmediate(reg, Immediate(-value), width);
  }
}

void Assembler::AddImmediate(Register dest, Register src, int64_t value) {
  if (dest == src) {
    AddImmediate(dest, value);
    return;
  }
  if (value == 0) {
    MoveRegister(dest, src);
    return;
  }
  if (Utils::IsInt(32, value)) {
    leaq(dest, Address(src, value));
    return;
  }
  LoadImmediate(dest, value);
  addq(dest, src);
}

void Assembler::AddImmediate(const Address& address, const Immediate& imm) {
  const int64_t value = imm.value();
  if (value == 0) {
    return;
  }
  if ((value > 0) || (value == kMinInt64)) {
    if (value == 1) {
      incq(address);
    } else {
      if (imm.is_int32()) {
        addq(address, imm);
      } else {
        LoadImmediate(TMP, imm);
        addq(address, TMP);
      }
    }
  } else {
    SubImmediate(address, Immediate(-value));
  }
}

void Assembler::SubImmediate(Register reg,
                             const Immediate& imm,
                             OperandSize width) {
  ASSERT(width == kFourBytes || width == kEightBytes);
  const int64_t value = imm.value();
  if (value == 0) {
    return;
  }
  if ((value > 0) || (value == kMinInt64) ||
      (value == kMinInt32 && width == kFourBytes)) {
    if (value == 1) {
      if (width == kFourBytes) {
        decl(reg);
      } else {
        decq(reg);
      }
    } else {
      if (imm.is_int32()) {
        if (width == kFourBytes) {
          subl(reg, imm);
        } else {
          subq(reg, imm);
        }
      } else {
        ASSERT(reg != TMP);
        ASSERT(width == kEightBytes);
        LoadImmediate(TMP, imm);
        subq(reg, TMP);
      }
    }
  } else {
    AddImmediate(reg, Immediate(-value), width);
  }
}

void Assembler::SubImmediate(const Address& address, const Immediate& imm) {
  const int64_t value = imm.value();
  if (value == 0) {
    return;
  }
  if ((value > 0) || (value == kMinInt64)) {
    if (value == 1) {
      decq(address);
    } else {
      if (imm.is_int32()) {
        subq(address, imm);
      } else {
        LoadImmediate(TMP, imm);
        subq(address, TMP);
      }
    }
  } else {
    AddImmediate(address, Immediate(-value));
  }
}

void Assembler::Drop(intptr_t stack_elements, Register tmp) {
  ASSERT(stack_elements >= 0);
  if (stack_elements <= 4) {
    for (intptr_t i = 0; i < stack_elements; i++) {
      popq(tmp);
    }
    return;
  }
  addq(RSP, Immediate(stack_elements * target::kWordSize));
}

bool Assembler::CanLoadFromObjectPool(const Object& object) const {
  ASSERT(IsOriginalObject(object));
  if (!constant_pool_allowed()) {
    return false;
  }

  DEBUG_ASSERT(IsNotTemporaryScopedHandle(object));
  ASSERT(IsInOldSpace(object));
  return true;
}

void Assembler::LoadWordFromPoolIndex(Register dst, intptr_t idx) {
  ASSERT(constant_pool_allowed());
  ASSERT(dst != PP);
  // PP is tagged on X64.
  const int32_t offset =
      target::ObjectPool::element_offset(idx) - kHeapObjectTag;
  // This sequence must be decodable by code_patcher_x64.cc.
  movq(dst, Address(PP, offset));
}

void Assembler::LoadIsolate(Register dst) {
  movq(dst, Address(THR, target::Thread::isolate_offset()));
}

void Assembler::LoadIsolateGroup(Register dst) {
  movq(dst, Address(THR, target::Thread::isolate_group_offset()));
}

void Assembler::LoadDispatchTable(Register dst) {
  movq(dst, Address(THR, target::Thread::dispatch_table_array_offset()));
}

void Assembler::LoadObjectHelper(Register dst,
                                 const Object& object,
                                 bool is_unique) {
  ASSERT(IsOriginalObject(object));

  // `is_unique == true` effectively means object has to be patchable.
  if (!is_unique) {
    intptr_t offset;
    if (target::CanLoadFromThread(object, &offset)) {
      movq(dst, Address(THR, offset));
      return;
    }
    if (target::IsSmi(object)) {
      LoadImmediate(dst, Immediate(target::ToRawSmi(object)));
      return;
    }
  }
  RELEASE_ASSERT(CanLoadFromObjectPool(object));
  const intptr_t index =
      is_unique ? object_pool_builder().AddObject(
                      object, ObjectPoolBuilderEntry::kPatchable)
                : object_pool_builder().FindObject(
                      object, ObjectPoolBuilderEntry::kNotPatchable);
  LoadWordFromPoolIndex(dst, index);
}

void Assembler::LoadObject(Register dst, const Object& object) {
  LoadObjectHelper(dst, object, false);
}

void Assembler::LoadUniqueObject(Register dst, const Object& object) {
  LoadObjectHelper(dst, object, true);
}

void Assembler::StoreObject(const Address& dst, const Object& object) {
  ASSERT(IsOriginalObject(object));

  intptr_t offset_from_thread;
  if (target::CanLoadFromThread(object, &offset_from_thread)) {
    movq(TMP, Address(THR, offset_from_thread));
    movq(dst, TMP);
  } else if (target::IsSmi(object)) {
    MoveImmediate(dst, Immediate(target::ToRawSmi(object)));
  } else {
    LoadObject(TMP, object);
    movq(dst, TMP);
  }
}

void Assembler::PushObject(const Object& object) {
  ASSERT(IsOriginalObject(object));

  intptr_t offset_from_thread;
  if (target::CanLoadFromThread(object, &offset_from_thread)) {
    pushq(Address(THR, offset_from_thread));
  } else if (target::IsSmi(object)) {
    PushImmediate(Immediate(target::ToRawSmi(object)));
  } else {
    LoadObject(TMP, object);
    pushq(TMP);
  }
}

void Assembler::CompareObject(Register reg, const Object& object) {
  ASSERT(IsOriginalObject(object));

  intptr_t offset_from_thread;
  if (target::CanLoadFromThread(object, &offset_from_thread)) {
    OBJ(cmp)(reg, Address(THR, offset_from_thread));
  } else if (target::IsSmi(object)) {
    CompareImmediate(reg, Immediate(target::ToRawSmi(object)), kObjectBytes);
  } else {
    RELEASE_ASSERT(CanLoadFromObjectPool(object));
    const intptr_t idx = object_pool_builder().FindObject(
        object, ObjectPoolBuilderEntry::kNotPatchable);
    const int32_t offset = target::ObjectPool::element_offset(idx);
    OBJ(cmp)(reg, Address(PP, offset - kHeapObjectTag));
  }
}

void Assembler::LoadImmediate(Register reg, const Immediate& imm) {
  if (imm.value() == 0) {
    xorl(reg, reg);
  } else if (imm.is_int32() || !constant_pool_allowed()) {
    movq(reg, imm);
  } else {
    const intptr_t idx = object_pool_builder().FindImmediate(imm.value());
    LoadWordFromPoolIndex(reg, idx);
  }
}

void Assembler::MoveImmediate(const Address& dst, const Immediate& imm) {
  if (imm.is_int32()) {
    movq(dst, imm);
  } else {
    LoadImmediate(TMP, imm);
    movq(dst, TMP);
  }
}

void Assembler::LoadDImmediate(FpuRegister dst, double immediate) {
  int64_t bits = bit_cast<int64_t>(immediate);
  if (bits == 0) {
    xorps(dst, dst);
  } else {
    intptr_t index = object_pool_builder().FindImmediate64(bits);
    LoadUnboxedDouble(
        dst, PP, target::ObjectPool::element_offset(index) - kHeapObjectTag);
  }
}

void Assembler::LoadQImmediate(FpuRegister dst, simd128_value_t immediate) {
  intptr_t index = object_pool_builder().FindImmediate128(immediate);
  movups(dst, Address(PP, target::ObjectPool::element_offset(index) -
                              kHeapObjectTag));
}

void Assembler::LoadCompressed(Register dest, const Address& slot) {
#if !defined(DART_COMPRESSED_POINTERS)
  movq(dest, slot);
#else
  movl(dest, slot);  // Zero-extension.
  addq(dest, Address(THR, target::Thread::heap_base_offset()));
#endif
}

void Assembler::LoadCompressedSmi(Register dest, const Address& slot) {
#if !defined(DART_COMPRESSED_POINTERS)
  movq(dest, slot);
#else
  movl(dest, slot);  // Zero-extension.
#endif
#if defined(DEBUG)
  Label done;
  BranchIfSmi(dest, &done, kNearJump);
  Stop("Expected Smi");
  Bind(&done);
#endif
}

void Assembler::StoreIntoObject(Register object,
                                const Address& dest,
                                Register value,
                                CanBeSmi can_be_smi,
                                MemoryOrder memory_order) {
  if (memory_order == kRelease) {
    StoreRelease(value, dest.base(), dest.disp32());
  } else {
    movq(dest, value);
  }
  StoreBarrier(object, value, can_be_smi);
}

void Assembler::StoreCompressedIntoObject(Register object,
                                          const Address& dest,
                                          Register value,
                                          CanBeSmi can_be_smi,
                                          MemoryOrder memory_order) {
  if (memory_order == kRelease) {
    StoreReleaseCompressed(value, dest.base(), dest.disp8());
  } else {
    OBJ(mov)(dest, value);
  }
  StoreBarrier(object, value, can_be_smi);
}

void Assembler::StoreBarrier(Register object,
                             Register value,
                             CanBeSmi can_be_smi) {
  // x.slot = x. Barrier should have be removed at the IL level.
  ASSERT(object != value);
  ASSERT(object != TMP);
  ASSERT(value != TMP);

  // In parallel, test whether
  //  - object is old and not remembered and value is new, or
  //  - object is old and value is old and not marked and concurrent marking is
  //    in progress
  // If so, call the WriteBarrier stub, which will either add object to the
  // store buffer (case 1) or add value to the marking stack (case 2).
  // Compare UntaggedObject::StorePointer.
  Label done;
  if (can_be_smi == kValueCanBeSmi) {
    BranchIfSmi(value, &done, kNearJump);
  }
  movb(ByteRegisterOf(TMP),
       FieldAddress(object, target::Object::tags_offset()));
  shrl(TMP, Immediate(target::UntaggedObject::kBarrierOverlapShift));
  andl(TMP, Address(THR, target::Thread::write_barrier_mask_offset()));
  testb(FieldAddress(value, target::Object::tags_offset()), TMP);
  j(ZERO, &done, kNearJump);

  Register object_for_call = object;
  if (value != kWriteBarrierValueReg) {
    // Unlikely. Only non-graph intrinsics.
    // TODO(rmacnak): Shuffle registers in intrinsics.
    pushq(kWriteBarrierValueReg);
    if (object == kWriteBarrierValueReg) {
      COMPILE_ASSERT(RBX != kWriteBarrierValueReg);
      COMPILE_ASSERT(RCX != kWriteBarrierValueReg);
      object_for_call = (value == RBX) ? RCX : RBX;
      pushq(object_for_call);
      movq(object_for_call, object);
    }
    movq(kWriteBarrierValueReg, value);
  }
  generate_invoke_write_barrier_wrapper_(object_for_call);
  if (value != kWriteBarrierValueReg) {
    if (object == kWriteBarrierValueReg) {
      popq(object_for_call);
    }
    popq(kWriteBarrierValueReg);
  }
  Bind(&done);
}

void Assembler::StoreIntoArray(Register object,
                               Register slot,
                               Register value,
                               CanBeSmi can_be_smi) {
  movq(Address(slot, 0), value);
  StoreIntoArrayBarrier(object, slot, value, can_be_smi);
}

void Assembler::StoreCompressedIntoArray(Register object,
                                         Register slot,
                                         Register value,
                                         CanBeSmi can_be_smi) {
  OBJ(mov)(Address(slot, 0), value);
  StoreIntoArrayBarrier(object, slot, value, can_be_smi);
}

void Assembler::StoreIntoArrayBarrier(Register object,
                                      Register slot,
                                      Register value,
                                      CanBeSmi can_be_smi) {
  ASSERT(object != TMP);
  ASSERT(value != TMP);
  ASSERT(slot != TMP);

  // In parallel, test whether
  //  - object is old and not remembered and value is new, or
  //  - object is old and value is old and not marked and concurrent marking is
  //    in progress
  // If so, call the WriteBarrier stub, which will either add object to the
  // store buffer (case 1) or add value to the marking stack (case 2).
  // Compare UntaggedObject::StorePointer.
  Label done;
  if (can_be_smi == kValueCanBeSmi) {
    BranchIfSmi(value, &done, kNearJump);
  }
  movb(ByteRegisterOf(TMP),
       FieldAddress(object, target::Object::tags_offset()));
  shrl(TMP, Immediate(target::UntaggedObject::kBarrierOverlapShift));
  andl(TMP, Address(THR, target::Thread::write_barrier_mask_offset()));
  testb(FieldAddress(value, target::Object::tags_offset()), TMP);
  j(ZERO, &done, kNearJump);

  if ((object != kWriteBarrierObjectReg) || (value != kWriteBarrierValueReg) ||
      (slot != kWriteBarrierSlotReg)) {
    // Spill and shuffle unimplemented. Currently StoreIntoArray is only used
    // from StoreIndexInstr, which gets these exact registers from the register
    // allocator.
    UNIMPLEMENTED();
  }

  generate_invoke_array_write_barrier_();

  Bind(&done);
}

void Assembler::StoreIntoObjectNoBarrier(Register object,
                                         const Address& dest,
                                         Register value,
                                         MemoryOrder memory_order) {
  if (memory_order == kRelease) {
    StoreRelease(value, dest.base(), dest.disp32());
  } else {
    movq(dest, value);
  }
#if defined(DEBUG)
  // We can't assert the incremental barrier is not needed here, only the
  // generational barrier. We sometimes omit the write barrier when 'value' is
  // a constant, but we don't eagerly mark 'value' and instead assume it is also
  // reachable via a constant pool, so it doesn't matter if it is not traced via
  // 'object'.
  Label done;
  BranchIfSmi(value, &done, kNearJump);
  testb(FieldAddress(value, target::Object::tags_offset()),
        Immediate(1 << target::UntaggedObject::kNewBit));
  j(ZERO, &done, Assembler::kNearJump);
  testb(FieldAddress(object, target::Object::tags_offset()),
        Immediate(1 << target::UntaggedObject::kOldAndNotRememberedBit));
  j(ZERO, &done, Assembler::kNearJump);
  Stop("Write barrier is required");
  Bind(&done);
#endif  // defined(DEBUG)
  // No store buffer update.
}

void Assembler::StoreCompressedIntoObjectNoBarrier(Register object,
                                                   const Address& dest,
                                                   Register value,
                                                   MemoryOrder memory_order) {
  if (memory_order == kRelease) {
    StoreReleaseCompressed(value, dest.base(), dest.disp8());
  } else {
    OBJ(mov)(dest, value);
  }
#if defined(DEBUG)
  // We can't assert the incremental barrier is not needed here, only the
  // generational barrier. We sometimes omit the write barrier when 'value' is
  // a constant, but we don't eagerly mark 'value' and instead assume it is also
  // reachable via a constant pool, so it doesn't matter if it is not traced via
  // 'object'.
  Label done;
  BranchIfSmi(value, &done, kNearJump);
  testb(FieldAddress(value, target::Object::tags_offset()),
        Immediate(1 << target::UntaggedObject::kNewBit));
  j(ZERO, &done, Assembler::kNearJump);
  testb(FieldAddress(object, target::Object::tags_offset()),
        Immediate(1 << target::UntaggedObject::kOldAndNotRememberedBit));
  j(ZERO, &done, Assembler::kNearJump);
  Stop("Write barrier is required");
  Bind(&done);
#endif  // defined(DEBUG)
  // No store buffer update.
}

void Assembler::StoreIntoObjectNoBarrier(Register object,
                                         const Address& dest,
                                         const Object& value,
                                         MemoryOrder memory_order) {
  if (memory_order == kRelease) {
    LoadObject(TMP, value);
    StoreIntoObjectNoBarrier(object, dest, TMP, memory_order);
  } else {
    StoreObject(dest, value);
  }
}

void Assembler::StoreCompressedIntoObjectNoBarrier(Register object,
                                                   const Address& dest,
                                                   const Object& value,
                                                   MemoryOrder memory_order) {
  LoadObject(TMP, value);
  StoreCompressedIntoObjectNoBarrier(object, dest, TMP, memory_order);
}

void Assembler::StoreInternalPointer(Register object,
                                     const Address& dest,
                                     Register value) {
  movq(dest, value);
}

void Assembler::StoreIntoSmiField(const Address& dest, Register value) {
#if defined(DEBUG)
  Label done;
  testq(value, Immediate(kHeapObjectTag));
  j(ZERO, &done);
  Stop("New value must be Smi.");
  Bind(&done);
#endif  // defined(DEBUG)
  movq(dest, value);
}

void Assembler::ZeroInitSmiField(const Address& dest) {
  Immediate zero(target::ToRawSmi(0));
  movq(dest, zero);
}

void Assembler::ZeroInitCompressedSmiField(const Address& dest) {
  Immediate zero(target::ToRawSmi(0));
  OBJ(mov)(dest, zero);
}

void Assembler::IncrementCompressedSmiField(const Address& dest,
                                            int64_t increment) {
  // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on
  // the length of this instruction sequence.
  Immediate inc_imm(target::ToRawSmi(increment));
  OBJ(add)(dest, inc_imm);
}

void Assembler::Bind(Label* label) {
  intptr_t bound = buffer_.Size();
  ASSERT(!label->IsBound());  // Labels can only be bound once.
  while (label->IsLinked()) {
    intptr_t position = label->LinkPosition();
    intptr_t next = buffer_.Load<int32_t>(position);
    buffer_.Store<int32_t>(position, bound - (position + 4));
    label->position_ = next;
  }
  while (label->HasNear()) {
    intptr_t position = label->NearPosition();
    intptr_t offset = bound - (position + 1);
    ASSERT(Utils::IsInt(8, offset));
    buffer_.Store<int8_t>(position, offset);
  }
  label->BindTo(bound);
}

void Assembler::LoadFromOffset(Register reg,
                               const Address& address,
                               OperandSize sz) {
  switch (sz) {
    case kByte:
      return movsxb(reg, address);
    case kUnsignedByte:
      return movzxb(reg, address);
    case kTwoBytes:
      return movsxw(reg, address);
    case kUnsignedTwoBytes:
      return movzxw(reg, address);
    case kFourBytes:
      return movsxd(reg, address);
    case kUnsignedFourBytes:
      return movl(reg, address);
    case kEightBytes:
      return movq(reg, address);
    default:
      UNREACHABLE();
      break;
  }
}

void Assembler::StoreToOffset(Register reg,
                              const Address& address,
                              OperandSize sz) {
  switch (sz) {
    case kByte:
    case kUnsignedByte:
      return movb(address, ByteRegisterOf(reg));
    case kTwoBytes:
    case kUnsignedTwoBytes:
      return movw(address, reg);
    case kFourBytes:
    case kUnsignedFourBytes:
      return movl(address, reg);
    case kEightBytes:
      return movq(address, reg);
    default:
      UNREACHABLE();
      break;
  }
}

void Assembler::ArithmeticShiftRightImmediate(Register reg, intptr_t shift) {
  sarq(reg, Immediate(shift));
}

void Assembler::CompareWords(Register reg1,
                             Register reg2,
                             intptr_t offset,
                             Register count,
                             Register temp,
                             Label* equals) {
  Label loop;
  Bind(&loop);
  decq(count);
  j(LESS, equals, Assembler::kNearJump);
  COMPILE_ASSERT(target::kWordSize == 8);
  movq(temp, FieldAddress(reg1, count, TIMES_8, offset));
  cmpq(temp, FieldAddress(reg2, count, TIMES_8, offset));
  BranchIf(EQUAL, &loop, Assembler::kNearJump);
}

void Assembler::EnterFrame(intptr_t frame_size) {
  if (prologue_offset_ == -1) {
    prologue_offset_ = CodeSize();
    Comment("PrologueOffset = %" Pd "", CodeSize());
  }
#ifdef DEBUG
  intptr_t check_offset = CodeSize();
#endif
  pushq(RBP);
  movq(RBP, RSP);
#ifdef DEBUG
  ProloguePattern pp(CodeAddress(check_offset));
  ASSERT(pp.IsValid());
#endif
  if (frame_size != 0) {
    Immediate frame_space(frame_size);
    subq(RSP, frame_space);
  }
}

void Assembler::LeaveFrame() {
  movq(RSP, RBP);
  popq(RBP);
}

void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
  // Reserve space for arguments and align frame before entering
  // the C++ world.
  if (frame_space != 0) {
    subq(RSP, Immediate(frame_space));
  }
  if (OS::ActivationFrameAlignment() > 1) {
    andq(RSP, Immediate(~(OS::ActivationFrameAlignment() - 1)));
  }
}

void Assembler::EmitEntryFrameVerification() {
#if defined(DEBUG)
  Label ok;
  leaq(RAX, Address(RBP, target::frame_layout.exit_link_slot_from_entry_fp *
                             target::kWordSize));
  cmpq(RAX, RSP);
  j(EQUAL, &ok);
  Stop("target::frame_layout.exit_link_slot_from_entry_fp mismatch");
  Bind(&ok);
#endif
}

void Assembler::PushRegisters(const RegisterSet& register_set) {
  const intptr_t xmm_regs_count = register_set.FpuRegisterCount();
  if (xmm_regs_count > 0) {
    AddImmediate(RSP, Immediate(-xmm_regs_count * kFpuRegisterSize));
    // Store XMM registers with the lowest register number at the lowest
    // address.
    intptr_t offset = 0;
    for (intptr_t i = 0; i < kNumberOfXmmRegisters; ++i) {
      XmmRegister xmm_reg = static_cast<XmmRegister>(i);
      if (register_set.ContainsFpuRegister(xmm_reg)) {
        movups(Address(RSP, offset), xmm_reg);
        offset += kFpuRegisterSize;
      }
    }
    ASSERT(offset == (xmm_regs_count * kFpuRegisterSize));
  }

  // The order in which the registers are pushed must match the order
  // in which the registers are encoded in the safe point's stack map.
  for (intptr_t i = kNumberOfCpuRegisters - 1; i >= 0; --i) {
    Register reg = static_cast<Register>(i);
    if (register_set.ContainsRegister(reg)) {
      pushq(reg);
    }
  }
}

void Assembler::PopRegisters(const RegisterSet& register_set) {
  for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) {
    Register reg = static_cast<Register>(i);
    if (register_set.ContainsRegister(reg)) {
      popq(reg);
    }
  }

  const intptr_t xmm_regs_count = register_set.FpuRegisterCount();
  if (xmm_regs_count > 0) {
    // XMM registers have the lowest register number at the lowest address.
    intptr_t offset = 0;
    for (intptr_t i = 0; i < kNumberOfXmmRegisters; ++i) {
      XmmRegister xmm_reg = static_cast<XmmRegister>(i);
      if (register_set.ContainsFpuRegister(xmm_reg)) {
        movups(xmm_reg, Address(RSP, offset));
        offset += kFpuRegisterSize;
      }
    }
    ASSERT(offset == (xmm_regs_count * kFpuRegisterSize));
    AddImmediate(RSP, Immediate(offset));
  }
}

void Assembler::PushRegistersInOrder(std::initializer_list<Register> regs) {
  for (Register reg : regs) {
    PushRegister(reg);
  }
}

static const RegisterSet kVolatileRegisterSet(
    CallingConventions::kVolatileCpuRegisters,
    CallingConventions::kVolatileXmmRegisters);

void Assembler::CallCFunction(Register reg, bool restore_rsp) {
  // Reserve shadow space for outgoing arguments.
  if (CallingConventions::kShadowSpaceBytes != 0) {
    subq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
  }
  call(reg);
  // Restore stack.
  if (restore_rsp && CallingConventions::kShadowSpaceBytes != 0) {
    addq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
  }
}
void Assembler::CallCFunction(Address address, bool restore_rsp) {
  // Reserve shadow space for outgoing arguments.
  if (CallingConventions::kShadowSpaceBytes != 0) {
    subq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
  }
  call(address);
  // Restore stack.
  if (restore_rsp && CallingConventions::kShadowSpaceBytes != 0) {
    addq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
  }
}

void Assembler::CallRuntime(const RuntimeEntry& entry,
                            intptr_t argument_count) {
  ASSERT(!entry.is_leaf());
  // Argument count is not checked here, but in the runtime entry for a more
  // informative error message.
  movq(RBX, compiler::Address(THR, entry.OffsetFromThread()));
  LoadImmediate(R10, compiler::Immediate(argument_count));
  call(Address(THR, target::Thread::call_to_runtime_entry_point_offset()));
}

#define __ assembler_->

LeafRuntimeScope::LeafRuntimeScope(Assembler* assembler,
                                   intptr_t frame_size,
                                   bool preserve_registers)
    : assembler_(assembler), preserve_registers_(preserve_registers) {
  __ Comment("EnterCallRuntimeFrame");
  __ EnterFrame(0);

  if (preserve_registers_) {
    // TODO(vegorov): avoid saving FpuTMP, it is used only as scratch.
    __ PushRegisters(kVolatileRegisterSet);
  } else {
    // These registers must always be preserved.
    ASSERT(IsCalleeSavedRegister(THR));
    ASSERT(IsCalleeSavedRegister(PP));
    ASSERT(IsCalleeSavedRegister(CODE_REG));
  }

  __ ReserveAlignedFrameSpace(frame_size);
}

void LeafRuntimeScope::Call(const RuntimeEntry& entry,
                            intptr_t argument_count) {
  ASSERT(entry.is_leaf());
  ASSERT(entry.argument_count() == argument_count);
  COMPILE_ASSERT(CallingConventions::kVolatileCpuRegisters & (1 << RAX));
  __ movq(RAX, compiler::Address(THR, entry.OffsetFromThread()));
  __ movq(compiler::Assembler::VMTagAddress(), RAX);
  __ CallCFunction(RAX);
  __ movq(compiler::Assembler::VMTagAddress(),
          compiler::Immediate(VMTag::kDartTagId));
}

LeafRuntimeScope::~LeafRuntimeScope() {
  if (preserve_registers_) {
    // RSP might have been modified to reserve space for arguments
    // and ensure proper alignment of the stack frame.
    // We need to restore it before restoring registers.
    const intptr_t kPushedCpuRegistersCount =
        RegisterSet::RegisterCount(CallingConventions::kVolatileCpuRegisters);
    const intptr_t kPushedXmmRegistersCount =
        RegisterSet::RegisterCount(CallingConventions::kVolatileXmmRegisters);
    const intptr_t kPushedRegistersSize =
        kPushedCpuRegistersCount * target::kWordSize +
        kPushedXmmRegistersCount * kFpuRegisterSize;

    __ leaq(RSP, Address(RBP, -kPushedRegistersSize));

    // TODO(vegorov): avoid saving FpuTMP, it is used only as scratch.
    __ PopRegisters(kVolatileRegisterSet);
  } else {
    const intptr_t kPushedRegistersSize =
        (target::frame_layout.dart_fixed_frame_size - 2) *
        target::kWordSize;  // From EnterStubFrame (excluding PC / FP)
    __ leaq(RSP, Address(RBP, -kPushedRegistersSize));
  }

  __ LeaveFrame();
}

#if defined(USING_THREAD_SANITIZER)
void Assembler::TsanLoadAcquire(Address addr) {
  LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
  leaq(CallingConventions::kArg1Reg, addr);
  rt.Call(kTsanLoadAcquireRuntimeEntry, /*argument_count=*/1);
}

void Assembler::TsanStoreRelease(Address addr) {
  LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
  leaq(CallingConventions::kArg1Reg, addr);
  rt.Call(kTsanStoreReleaseRuntimeEntry, /*argument_count=*/1);
}
#endif

void Assembler::RestoreCodePointer() {
  movq(CODE_REG,
       Address(RBP, target::frame_layout.code_from_fp * target::kWordSize));
}

void Assembler::LoadPoolPointer(Register pp) {
  // Load new pool pointer.
  CheckCodePointer();
  movq(pp, FieldAddress(CODE_REG, target::Code::object_pool_offset()));
  set_constant_pool_allowed(pp == PP);
}

void Assembler::EnterDartFrame(intptr_t frame_size, Register new_pp) {
  ASSERT(!constant_pool_allowed());
  EnterFrame(0);
  if (!FLAG_precompiled_mode) {
    pushq(CODE_REG);
    pushq(PP);
    if (new_pp == kNoRegister) {
      LoadPoolPointer(PP);
    } else {
      movq(PP, new_pp);
    }
  }
  set_constant_pool_allowed(true);
  if (frame_size != 0) {
    subq(RSP, Immediate(frame_size));
  }
}

void Assembler::LeaveDartFrame() {
  // Restore caller's PP register that was pushed in EnterDartFrame.
  if (!FLAG_precompiled_mode) {
    movq(PP, Address(RBP, (target::frame_layout.saved_caller_pp_from_fp *
                           target::kWordSize)));
  }
  set_constant_pool_allowed(false);
  LeaveFrame();
}

void Assembler::CheckCodePointer() {
#ifdef DEBUG
  if (!FLAG_check_code_pointer) {
    return;
  }
  Comment("CheckCodePointer");
  Label cid_ok, instructions_ok;
  pushq(RAX);
  LoadClassId(RAX, CODE_REG);
  cmpq(RAX, Immediate(kCodeCid));
  j(EQUAL, &cid_ok);
  int3();
  Bind(&cid_ok);
  {
    const intptr_t kRIPRelativeLeaqSize = 7;
    const intptr_t header_to_entry_offset =
        (target::Instructions::HeaderSize() - kHeapObjectTag);
    const intptr_t header_to_rip_offset =
        CodeSize() + kRIPRelativeLeaqSize + header_to_entry_offset;
    leaq(RAX, Address::AddressRIPRelative(-header_to_rip_offset));
    ASSERT(CodeSize() == (header_to_rip_offset - header_to_entry_offset));
  }
  cmpq(RAX, FieldAddress(CODE_REG, target::Code::instructions_offset()));
  j(EQUAL, &instructions_ok);
  int3();
  Bind(&instructions_ok);
  popq(RAX);
#endif
}

// On entry to a function compiled for OSR, the caller's frame pointer, the
// stack locals, and any copied parameters are already in place.  The frame
// pointer is already set up.  The PC marker is not correct for the
// optimized function and there may be extra space for spill slots to
// allocate.
void Assembler::EnterOsrFrame(intptr_t extra_size) {
  ASSERT(!constant_pool_allowed());
  if (prologue_offset_ == -1) {
    Comment("PrologueOffset = %" Pd "", CodeSize());
    prologue_offset_ = CodeSize();
  }
  RestoreCodePointer();
  LoadPoolPointer();

  if (extra_size != 0) {
    subq(RSP, Immediate(extra_size));
  }
}

void Assembler::EnterStubFrame() {
  EnterDartFrame(0, kNoRegister);
}

void Assembler::LeaveStubFrame() {
  LeaveDartFrame();
}

void Assembler::EnterCFrame(intptr_t frame_space) {
  // Already saved.
  COMPILE_ASSERT(IsCalleeSavedRegister(THR));
  COMPILE_ASSERT(IsCalleeSavedRegister(PP));

  EnterFrame(0);
  ReserveAlignedFrameSpace(frame_space);
}

void Assembler::LeaveCFrame() {
  LeaveFrame();
}

// RDX receiver, RBX ICData entries array
// Preserve R10 (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntryJIT() {
  has_monomorphic_entry_ = true;
  intptr_t start = CodeSize();
  Label have_cid, miss;
  Bind(&miss);
  jmp(Address(THR, target::Thread::switchable_call_miss_entry_offset()));

  // Ensure the monomorphic entry is 2-byte aligned (so GC can see them if we
  // store them in ICData / MegamorphicCache arrays)
  nop(1);

  Comment("MonomorphicCheckedEntry");
  ASSERT_EQUAL(CodeSize() - start,
               target::Instructions::kMonomorphicEntryOffsetJIT);
  ASSERT((CodeSize() & kSmiTagMask) == kSmiTag);

  const intptr_t cid_offset = target::Array::element_offset(0);
  const intptr_t count_offset = target::Array::element_offset(1);

  LoadTaggedClassIdMayBeSmi(RAX, RDX);

  OBJ(cmp)(RAX, FieldAddress(RBX, cid_offset));
  j(NOT_EQUAL, &miss, Assembler::kNearJump);
  OBJ(add)(FieldAddress(RBX, count_offset), Immediate(target::ToRawSmi(1)));
  xorq(R10, R10);  // GC-safe for OptimizeInvokedFunction.
#if defined(DART_COMPRESSED_POINTERS)
  nop(4);
#else
  nop(1);
#endif

  // Fall through to unchecked entry.
  ASSERT_EQUAL(CodeSize() - start,
               target::Instructions::kPolymorphicEntryOffsetJIT);
  ASSERT(((CodeSize() - start) & kSmiTagMask) == kSmiTag);
}

// RBX - input: class id smi
// RDX - input: receiver object
void Assembler::MonomorphicCheckedEntryAOT() {
  has_monomorphic_entry_ = true;
  intptr_t start = CodeSize();
  Label have_cid, miss;
  Bind(&miss);
  jmp(Address(THR, target::Thread::switchable_call_miss_entry_offset()));

  // Ensure the monomorphic entry is 2-byte aligned (so GC can see them if we
  // store them in ICData / MegamorphicCache arrays)
  nop(1);

  Comment("MonomorphicCheckedEntry");
  ASSERT_EQUAL(CodeSize() - start,
               target::Instructions::kMonomorphicEntryOffsetAOT);
  ASSERT((CodeSize() & kSmiTagMask) == kSmiTag);

  SmiUntag(RBX);
  LoadClassId(RAX, RDX);
  cmpq(RAX, RBX);
  j(NOT_EQUAL, &miss, Assembler::kNearJump);

  // Ensure the unchecked entry is 2-byte aligned (so GC can see them if we
  // store them in ICData / MegamorphicCache arrays).
#if defined(DART_COMPRESSED_POINTERS)
  nop(1);
#endif

  // Fall through to unchecked entry.
  ASSERT_EQUAL(CodeSize() - start,
               target::Instructions::kPolymorphicEntryOffsetAOT);
  ASSERT(((CodeSize() - start) & kSmiTagMask) == kSmiTag);
}

void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
  has_monomorphic_entry_ = true;
  while (CodeSize() < target::Instructions::kMonomorphicEntryOffsetJIT) {
    int3();
  }
  jmp(label);
  while (CodeSize() < target::Instructions::kPolymorphicEntryOffsetJIT) {
    int3();
  }
}

void Assembler::CombineHashes(Register dst, Register other) {
  // hash += other_hash
  addl(dst, other);
  // hash += hash << 10
  movl(other, dst);
  shll(other, Immediate(10));
  addl(dst, other);
  // hash ^= hash >> 6
  movl(other, dst);
  shrl(other, Immediate(6));
  xorl(dst, other);
}

void Assembler::FinalizeHashForSize(intptr_t bit_size,
                                    Register dst,
                                    Register scratch) {
  ASSERT(bit_size > 0);  // Can't avoid returning 0 if there are no hash bits!
  // While any 32-bit hash value fits in X bits, where X > 32, the caller may
  // reasonably expect that the returned values fill the entire bit space.
  ASSERT(bit_size <= kBitsPerInt32);
  ASSERT(scratch != kNoRegister);
  // hash += hash << 3;
  movl(scratch, dst);
  shll(scratch, Immediate(3));
  addl(dst, scratch);
  // hash ^= hash >> 11;  // Logical shift, unsigned hash.
  movl(scratch, dst);
  shrl(scratch, Immediate(11));
  xorl(dst, scratch);
  // hash += hash << 15;
  movl(scratch, dst);
  shll(scratch, Immediate(15));
  addl(dst, scratch);
  // Size to fit.
  if (bit_size < kBitsPerInt32) {
    andl(dst, Immediate(Utils::NBitMask(bit_size)));
  }
  // return (hash == 0) ? 1 : hash;
  Label done;
  j(NOT_ZERO, &done, kNearJump);
  incl(dst);
  Bind(&done);
}

#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
                                     Label* trace,
                                     Register temp_reg,
                                     JumpDistance distance) {
  ASSERT(cid > 0);

  if (temp_reg == kNoRegister) {
    temp_reg = TMP;
  }
  LoadIsolateGroup(temp_reg);
  movq(temp_reg, Address(temp_reg, target::IsolateGroup::class_table_offset()));
  movq(temp_reg,
       Address(temp_reg,
               target::ClassTable::allocation_tracing_state_table_offset()));
  cmpb(Address(temp_reg,
               target::ClassTable::AllocationTracingStateSlotOffsetFor(cid)),
       Immediate(0));
  // We are tracing for this class, jump to the trace label which will use
  // the allocation stub.
  j(NOT_ZERO, trace, distance);
}
#endif  // !PRODUCT

void Assembler::TryAllocateObject(intptr_t cid,
                                  intptr_t instance_size,
                                  Label* failure,
                                  JumpDistance distance,
                                  Register instance_reg,
                                  Register temp_reg) {
  ASSERT(failure != NULL);
  ASSERT(instance_size != 0);
  ASSERT(Utils::IsAligned(instance_size,
                          target::ObjectAlignment::kObjectAlignment));
  if (FLAG_inline_alloc &&
      target::Heap::IsAllocatableInNewSpace(instance_size)) {
    // If this allocation is traced, program will jump to failure path
    // (i.e. the allocation stub) which will allocate the object and trace the
    // allocation call site.
    NOT_IN_PRODUCT(MaybeTraceAllocation(cid, failure, temp_reg, distance));
    movq(instance_reg, Address(THR, target::Thread::top_offset()));
    addq(instance_reg, Immediate(instance_size));
    // instance_reg: potential next object start.
    cmpq(instance_reg, Address(THR, target::Thread::end_offset()));
    j(ABOVE_EQUAL, failure, distance);
    // Successfully allocated the object, now update top to point to
    // next object start and store the class in the class field of object.
    movq(Address(THR, target::Thread::top_offset()), instance_reg);
    ASSERT(instance_size >= kHeapObjectTag);
    AddImmediate(instance_reg, Immediate(kHeapObjectTag - instance_size));
    const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
    MoveImmediate(FieldAddress(instance_reg, target::Object::tags_offset()),
                  Immediate(tags));
  } else {
    jmp(failure);
  }
}

void Assembler::TryAllocateArray(intptr_t cid,
                                 intptr_t instance_size,
                                 Label* failure,
                                 JumpDistance distance,
                                 Register instance,
                                 Register end_address,
                                 Register temp) {
  ASSERT(failure != NULL);
  if (FLAG_inline_alloc &&
      target::Heap::IsAllocatableInNewSpace(instance_size)) {
    // If this allocation is traced, program will jump to failure path
    // (i.e. the allocation stub) which will allocate the object and trace the
    // allocation call site.
    NOT_IN_PRODUCT(MaybeTraceAllocation(cid, failure, temp, distance));
    movq(instance, Address(THR, target::Thread::top_offset()));
    movq(end_address, instance);

    addq(end_address, Immediate(instance_size));
    j(CARRY, failure);

    // Check if the allocation fits into the remaining space.
    // instance: potential new object start.
    // end_address: potential next object start.
    cmpq(end_address, Address(THR, target::Thread::end_offset()));
    j(ABOVE_EQUAL, failure);

    // Successfully allocated the object(s), now update top to point to
    // next object start and initialize the object.
    movq(Address(THR, target::Thread::top_offset()), end_address);
    addq(instance, Immediate(kHeapObjectTag));

    // Initialize the tags.
    // instance: new object start as a tagged pointer.
    const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
    movq(FieldAddress(instance, target::Object::tags_offset()),
         Immediate(tags));
  } else {
    jmp(failure);
  }
}

void Assembler::CopyMemoryWords(Register src,
                                Register dst,
                                Register size,
                                Register temp) {
  // This loop is equivalent to
  //   shrq(size, Immediate(target::kWordSizeLog2));
  //   rep_movsq()
  // but shows better performance on certain micro-benchmarks.
  Label loop, done;
  cmpq(size, Immediate(0));
  j(EQUAL, &done, kNearJump);
  Bind(&loop);
  movq(temp, Address(src, 0));
  addq(src, Immediate(target::kWordSize));
  movq(Address(dst, 0), temp);
  addq(dst, Immediate(target::kWordSize));
  subq(size, Immediate(target::kWordSize));
  j(NOT_ZERO, &loop, kNearJump);
  Bind(&done);
}

void Assembler::GenerateUnRelocatedPcRelativeCall(intptr_t offset_into_target) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  buffer_.Emit<uint8_t>(0xe8);
  buffer_.Emit<int32_t>(0);

  PcRelativeCallPattern pattern(buffer_.contents() + buffer_.Size() -
                                PcRelativeCallPattern::kLengthInBytes);
  pattern.set_distance(offset_into_target);
}

void Assembler::GenerateUnRelocatedPcRelativeTailCall(
    intptr_t offset_into_target) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  buffer_.Emit<uint8_t>(0xe9);
  buffer_.Emit<int32_t>(0);

  PcRelativeCallPattern pattern(buffer_.contents() + buffer_.Size() -
                                PcRelativeCallPattern::kLengthInBytes);
  pattern.set_distance(offset_into_target);
}

void Assembler::Align(int alignment, intptr_t offset) {
  ASSERT(Utils::IsPowerOfTwo(alignment));
  intptr_t pos = offset + buffer_.GetPosition();
  int mod = pos & (alignment - 1);
  if (mod == 0) {
    return;
  }
  intptr_t bytes_needed = alignment - mod;
  while (bytes_needed > MAX_NOP_SIZE) {
    nop(MAX_NOP_SIZE);
    bytes_needed -= MAX_NOP_SIZE;
  }
  if (bytes_needed != 0) {
    nop(bytes_needed);
  }
  ASSERT(((offset + buffer_.GetPosition()) & (alignment - 1)) == 0);
}

void Assembler::EmitOperand(int rm, const Operand& operand) {
  ASSERT(rm >= 0 && rm < 8);
  const intptr_t length = operand.length_;
  ASSERT(length > 0);
  // Emit the ModRM byte updated with the given RM value.
  ASSERT((operand.encoding_[0] & 0x38) == 0);
  EmitUint8(operand.encoding_[0] + (rm << 3));
  // Emit the rest of the encoded operand.
  for (intptr_t i = 1; i < length; i++) {
    EmitUint8(operand.encoding_[i]);
  }
}

void Assembler::EmitRegisterOperand(int rm, int reg) {
  Operand operand;
  operand.SetModRM(3, static_cast<Register>(reg));
  EmitOperand(rm, operand);
}

void Assembler::EmitImmediate(const Immediate& imm) {
  if (imm.is_int32()) {
    EmitInt32(static_cast<int32_t>(imm.value()));
  } else {
    EmitInt64(imm.value());
  }
}

void Assembler::EmitSignExtendedInt8(int rm,
                                     const Operand& operand,
                                     const Immediate& immediate) {
  EmitUint8(0x83);
  EmitOperand(rm, operand);
  EmitUint8(immediate.value() & 0xFF);
}

void Assembler::EmitComplex(int rm,
                            const Operand& operand,
                            const Immediate& immediate) {
  ASSERT(rm >= 0 && rm < 8);
  ASSERT(immediate.is_int32());
  if (immediate.is_int8()) {
    EmitSignExtendedInt8(rm, operand, immediate);
  } else if (operand.IsRegister(RAX)) {
    // Use short form if the destination is rax.
    EmitUint8(0x05 + (rm << 3));
    EmitImmediate(immediate);
  } else {
    EmitUint8(0x81);
    EmitOperand(rm, operand);
    EmitImmediate(immediate);
  }
}

void Assembler::EmitLabel(Label* label, intptr_t instruction_size) {
  if (label->IsBound()) {
    intptr_t offset = label->Position() - buffer_.Size();
    ASSERT(offset <= 0);
    EmitInt32(offset - instruction_size);
  } else {
    EmitLabelLink(label);
  }
}

void Assembler::EmitLabelLink(Label* label) {
  ASSERT(!label->IsBound());
  intptr_t position = buffer_.Size();
  EmitInt32(label->position_);
  label->LinkTo(position);
}

void Assembler::EmitNearLabelLink(Label* label) {
  ASSERT(!label->IsBound());
  intptr_t position = buffer_.Size();
  EmitUint8(0);
  label->NearLinkTo(position);
}

void Assembler::EmitGenericShift(bool wide,
                                 int rm,
                                 Register reg,
                                 const Immediate& imm) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  ASSERT(imm.is_int8());
  if (wide) {
    EmitRegisterREX(reg, REX_W);
  } else {
    EmitRegisterREX(reg, REX_NONE);
  }
  if (imm.value() == 1) {
    EmitUint8(0xD1);
    EmitOperand(rm, Operand(reg));
  } else {
    EmitUint8(0xC1);
    EmitOperand(rm, Operand(reg));
    EmitUint8(imm.value() & 0xFF);
  }
}

void Assembler::EmitGenericShift(bool wide,
                                 int rm,
                                 Register operand,
                                 Register shifter) {
  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
  ASSERT(shifter == RCX);
  EmitRegisterREX(operand, wide ? REX_W : REX_NONE);
  EmitUint8(0xD3);
  EmitOperand(rm, Operand(operand));
}

void Assembler::ExtractClassIdFromTags(Register result, Register tags) {
  ASSERT(target::UntaggedObject::kClassIdTagPos == 12);
  ASSERT(target::UntaggedObject::kClassIdTagSize == 20);
  movl(result, tags);
  shrl(result, Immediate(12));
}

void Assembler::ExtractInstanceSizeFromTags(Register result, Register tags) {
  ASSERT(target::UntaggedObject::kSizeTagPos == 8);
  ASSERT(target::UntaggedObject::kSizeTagSize == 4);
  movzxw(result, tags);
  shrl(result, Immediate(target::UntaggedObject::kSizeTagPos -
                         target::ObjectAlignment::kObjectAlignmentLog2));
  AndImmediate(result,
               Immediate(Utils::NBitMask(target::UntaggedObject::kSizeTagSize)
                         << target::ObjectAlignment::kObjectAlignmentLog2));
}

void Assembler::LoadClassId(Register result, Register object) {
  ASSERT(target::UntaggedObject::kClassIdTagPos == 12);
  ASSERT(target::UntaggedObject::kClassIdTagSize == 20);
  movl(result, FieldAddress(object, target::Object::tags_offset()));
  shrl(result, Immediate(target::UntaggedObject::kClassIdTagPos));
}

void Assembler::LoadClassById(Register result, Register class_id) {
  ASSERT(result != class_id);
  const intptr_t table_offset =
      target::IsolateGroup::cached_class_table_table_offset();

  LoadIsolateGroup(result);
  movq(result, Address(result, table_offset));
  movq(result, Address(result, class_id, TIMES_8, 0));
}

void Assembler::CompareClassId(Register object,
                               intptr_t class_id,
                               Register scratch) {
  LoadClassId(TMP, object);
  cmpl(TMP, Immediate(class_id));
}

void Assembler::SmiUntagOrCheckClass(Register object,
                                     intptr_t class_id,
                                     Label* is_smi) {
#if !defined(DART_COMPRESSED_POINTERS)
  ASSERT(kSmiTagShift == 1);
  ASSERT(target::UntaggedObject::kClassIdTagPos == 12);
  ASSERT(target::UntaggedObject::kClassIdTagSize == 20);
  // Untag optimistically. Tag bit is shifted into the CARRY.
  SmiUntag(object);
  j(NOT_CARRY, is_smi, kNearJump);
  // Load cid: can't use LoadClassId, object is untagged. Use TIMES_2 scale
  // factor in the addressing mode to compensate for this.
  movl(TMP, Address(object, TIMES_2,
                    target::Object::tags_offset() + kHeapObjectTag));
  shrl(TMP, Immediate(target::UntaggedObject::kClassIdTagPos));
  cmpl(TMP, Immediate(class_id));
#else
  // Cannot speculatively untag compressed Smis because it erases upper address
  // bits.
  UNREACHABLE();
#endif
}

void Assembler::LoadClassIdMayBeSmi(Register result, Register object) {
  Label smi;

  if (result == object) {
    Label join;

    testq(object, Immediate(kSmiTagMask));
    j(EQUAL, &smi, Assembler::kNearJump);
    LoadClassId(result, object);
    jmp(&join, Assembler::kNearJump);

    Bind(&smi);
    movq(result, Immediate(kSmiCid));

    Bind(&join);
  } else {
    testq(object, Immediate(kSmiTagMask));
    movq(result, Immediate(kSmiCid));
    j(EQUAL, &smi, Assembler::kNearJump);
    LoadClassId(result, object);

    Bind(&smi);
  }
}

void Assembler::LoadTaggedClassIdMayBeSmi(Register result, Register object) {
  Label smi;

  if (result == object) {
    Label join;

    testq(object, Immediate(kSmiTagMask));
    j(EQUAL, &smi, Assembler::kNearJump);
    LoadClassId(result, object);
    SmiTag(result);
    jmp(&join, Assembler::kNearJump);

    Bind(&smi);
    movq(result, Immediate(target::ToRawSmi(kSmiCid)));

    Bind(&join);
  } else {
    testq(object, Immediate(kSmiTagMask));
    movq(result, Immediate(target::ToRawSmi(kSmiCid)));
    j(EQUAL, &smi, Assembler::kNearJump);
    LoadClassId(result, object);
    SmiTag(result);

    Bind(&smi);
  }
}

void Assembler::EnsureHasClassIdInDEBUG(intptr_t cid,
                                        Register src,
                                        Register scratch,
                                        bool can_be_null) {
#if defined(DEBUG)
  Comment("Check that object in register has cid %" Pd "", cid);
  Label matches;
  LoadClassIdMayBeSmi(scratch, src);
  CompareImmediate(scratch, cid);
  BranchIf(EQUAL, &matches, Assembler::kNearJump);
  if (can_be_null) {
    CompareImmediate(scratch, kNullCid);
    BranchIf(EQUAL, &matches, Assembler::kNearJump);
  }
  Breakpoint();
  Bind(&matches);
#endif
}

Address Assembler::VMTagAddress() {
  return Address(THR, target::Thread::vm_tag_offset());
}

Address Assembler::ElementAddressForIntIndex(bool is_external,
                                             intptr_t cid,
                                             intptr_t index_scale,
                                             Register array,
                                             intptr_t index) {
  if (is_external) {
    return Address(array, index * index_scale);
  } else {
    const int64_t disp = static_cast<int64_t>(index) * index_scale +
                         target::Instance::DataOffsetFor(cid);
    ASSERT(Utils::IsInt(32, disp));
    return FieldAddress(array, static_cast<int32_t>(disp));
  }
}

static ScaleFactor ToScaleFactor(intptr_t index_scale, bool index_unboxed) {
  if (index_unboxed) {
    switch (index_scale) {
      case 1:
        return TIMES_1;
      case 2:
        return TIMES_2;
      case 4:
        return TIMES_4;
      case 8:
        return TIMES_8;
      case 16:
        return TIMES_16;
      default:
        UNREACHABLE();
        return TIMES_1;
    }
  } else {
    // Note that index is expected smi-tagged, (i.e, times 2) for all arrays
    // with index scale factor > 1. E.g., for Uint8Array and OneByteString the
    // index is expected to be untagged before accessing.
    ASSERT(kSmiTagShift == 1);
    switch (index_scale) {
      case 1:
        return TIMES_1;
      case 2:
        return TIMES_1;
      case 4:
        return TIMES_2;
      case 8:
        return TIMES_4;
      case 16:
        return TIMES_8;
      default:
        UNREACHABLE();
        return TIMES_1;
    }
  }
}

Address Assembler::ElementAddressForRegIndex(bool is_external,
                                             intptr_t cid,
                                             intptr_t index_scale,
                                             bool index_unboxed,
                                             Register array,
                                             Register index) {
  if (is_external) {
    return Address(array, index, ToScaleFactor(index_scale, index_unboxed), 0);
  } else {
    return FieldAddress(array, index, ToScaleFactor(index_scale, index_unboxed),
                        target::Instance::DataOffsetFor(cid));
  }
}

void Assembler::RangeCheck(Register value,
                           Register temp,
                           intptr_t low,
                           intptr_t high,
                           RangeCheckCondition condition,
                           Label* target) {
  auto cc = condition == kIfInRange ? BELOW_EQUAL : ABOVE;
  Register to_check = value;
  if (temp != kNoRegister) {
    movq(temp, value);
    to_check = temp;
  }
  subq(to_check, Immediate(low));
  cmpq(to_check, Immediate(high - low));
  j(cc, target);
}

}  // namespace compiler
}  // namespace dart

#endif  // defined(TARGET_ARCH_X64)