| // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | |
| // for details. All rights reserved. Use of this source code is governed by a | |
| // BSD-style license that can be found in the LICENSE file. | |
| #ifndef RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_X64_H_ | |
| #define RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_X64_H_ | |
| #if defined(DART_PRECOMPILED_RUNTIME) | |
| #error "AOT runtime should not use compiler sources (including header files)" | |
| #endif // defined(DART_PRECOMPILED_RUNTIME) | |
| #ifndef RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_H_ | |
| #error Do not include assembler_x64.h directly; use assembler.h instead. | |
| #endif | |
| #include <functional> | |
| #include "platform/assert.h" | |
| #include "platform/utils.h" | |
| #include "vm/compiler/assembler/assembler_base.h" | |
| #include "vm/constants.h" | |
| #include "vm/constants_x86.h" | |
| #include "vm/hash_map.h" | |
| #include "vm/pointer_tagging.h" | |
| namespace dart { | |
| // Forward declarations. | |
| class FlowGraphCompiler; | |
| class RegisterSet; | |
| namespace compiler { | |
| class Immediate : public ValueObject { | |
| public: | |
| explicit Immediate(int64_t value) : value_(value) {} | |
| Immediate(const Immediate& other) : ValueObject(), value_(other.value_) {} | |
| int64_t value() const { return value_; } | |
| bool is_int8() const { return Utils::IsInt(8, value_); } | |
| bool is_uint8() const { return Utils::IsUint(8, value_); } | |
| bool is_int16() const { return Utils::IsInt(16, value_); } | |
| bool is_uint16() const { return Utils::IsUint(16, value_); } | |
| bool is_int32() const { return Utils::IsInt(32, value_); } | |
| bool is_uint32() const { return Utils::IsUint(32, value_); } | |
| private: | |
| const int64_t value_; | |
| // TODO(5411081): Add DISALLOW_COPY_AND_ASSIGN(Immediate) once the mac | |
| // build issue is resolved. | |
| // And remove the unnecessary copy constructor. | |
| }; | |
| class Operand : public ValueObject { | |
| public: | |
| uint8_t rex() const { return rex_; } | |
| uint8_t mod() const { return (encoding_at(0) >> 6) & 3; } | |
| Register rm() const { | |
| int rm_rex = (rex_ & REX_B) << 3; | |
| return static_cast<Register>(rm_rex + (encoding_at(0) & 7)); | |
| } | |
| ScaleFactor scale() const { | |
| return static_cast<ScaleFactor>((encoding_at(1) >> 6) & 3); | |
| } | |
| Register index() const { | |
| int index_rex = (rex_ & REX_X) << 2; | |
| return static_cast<Register>(index_rex + ((encoding_at(1) >> 3) & 7)); | |
| } | |
| Register base() const { | |
| int base_rex = (rex_ & REX_B) << 3; | |
| return static_cast<Register>(base_rex + (encoding_at(1) & 7)); | |
| } | |
| int8_t disp8() const { | |
| ASSERT(length_ >= 2); | |
| return static_cast<int8_t>(encoding_[length_ - 1]); | |
| } | |
| int32_t disp32() const { | |
| ASSERT(length_ >= 5); | |
| return bit_copy<int32_t>(encoding_[length_ - 4]); | |
| } | |
| Operand(const Operand& other) | |
| : ValueObject(), length_(other.length_), rex_(other.rex_) { | |
| memmove(&encoding_[0], &other.encoding_[0], other.length_); | |
| } | |
| Operand& operator=(const Operand& other) { | |
| length_ = other.length_; | |
| rex_ = other.rex_; | |
| memmove(&encoding_[0], &other.encoding_[0], other.length_); | |
| return *this; | |
| } | |
| bool Equals(const Operand& other) const { | |
| if (length_ != other.length_) return false; | |
| if (rex_ != other.rex_) return false; | |
| for (uint8_t i = 0; i < length_; i++) { | |
| if (encoding_[i] != other.encoding_[i]) return false; | |
| } | |
| return true; | |
| } | |
| protected: | |
| Operand() : length_(0), rex_(REX_NONE) {} // Needed by subclass Address. | |
| void SetModRM(int mod, Register rm) { | |
| ASSERT((mod & ~3) == 0); | |
| if ((rm > 7) && !((rm == R12) && (mod != 3))) { | |
| rex_ |= REX_B; | |
| } | |
| encoding_[0] = (mod << 6) | (rm & 7); | |
| length_ = 1; | |
| } | |
| void SetSIB(ScaleFactor scale, Register index, Register base) { | |
| ASSERT(length_ == 1); | |
| ASSERT((scale & ~3) == 0); | |
| if (base > 7) { | |
| ASSERT((rex_ & REX_B) == 0); // Must not have REX.B already set. | |
| rex_ |= REX_B; | |
| } | |
| if (index > 7) rex_ |= REX_X; | |
| encoding_[1] = (scale << 6) | ((index & 7) << 3) | (base & 7); | |
| length_ = 2; | |
| } | |
| void SetDisp8(int8_t disp) { | |
| ASSERT(length_ == 1 || length_ == 2); | |
| encoding_[length_++] = static_cast<uint8_t>(disp); | |
| } | |
| void SetDisp32(int32_t disp) { | |
| ASSERT(length_ == 1 || length_ == 2); | |
| memmove(&encoding_[length_], &disp, sizeof(disp)); | |
| length_ += sizeof(disp); | |
| } | |
| private: | |
| uint8_t length_; | |
| uint8_t rex_; | |
| uint8_t encoding_[6]; | |
| explicit Operand(Register reg) : rex_(REX_NONE) { SetModRM(3, reg); } | |
| // Get the operand encoding byte at the given index. | |
| uint8_t encoding_at(intptr_t index) const { | |
| ASSERT(index >= 0 && index < length_); | |
| return encoding_[index]; | |
| } | |
| // Returns whether or not this operand is really the given register in | |
| // disguise. Used from the assembler to generate better encodings. | |
| bool IsRegister(Register reg) const { | |
| return ((reg > 7 ? 1 : 0) == (rex_ & REX_B)) // REX.B match. | |
| && ((encoding_at(0) & 0xF8) == 0xC0) // Addressing mode is register. | |
| && ((encoding_at(0) & 0x07) == reg); // Register codes match. | |
| } | |
| friend class Assembler; | |
| }; | |
| class Address : public Operand { | |
| public: | |
| Address(Register base, int32_t disp) { | |
| if ((disp == 0) && ((base & 7) != RBP)) { | |
| SetModRM(0, base); | |
| if ((base & 7) == RSP) { | |
| SetSIB(TIMES_1, RSP, base); | |
| } | |
| } else if (Utils::IsInt(8, disp)) { | |
| SetModRM(1, base); | |
| if ((base & 7) == RSP) { | |
| SetSIB(TIMES_1, RSP, base); | |
| } | |
| SetDisp8(disp); | |
| } else { | |
| SetModRM(2, base); | |
| if ((base & 7) == RSP) { | |
| SetSIB(TIMES_1, RSP, base); | |
| } | |
| SetDisp32(disp); | |
| } | |
| } | |
| // This addressing mode does not exist. | |
| Address(Register base, Register r); | |
| Address(Register index, ScaleFactor scale, int32_t disp) { | |
| ASSERT(index != RSP); // Illegal addressing mode. | |
| ASSERT(scale != TIMES_16); // Unsupported scale factor. | |
| SetModRM(0, RSP); | |
| SetSIB(scale, index, RBP); | |
| SetDisp32(disp); | |
| } | |
| // This addressing mode does not exist. | |
| Address(Register index, ScaleFactor scale, Register r); | |
| Address(Register base, Register index, ScaleFactor scale, int32_t disp) { | |
| ASSERT(index != RSP); // Illegal addressing mode. | |
| ASSERT(scale != TIMES_16); // Unsupported scale factor. | |
| if ((disp == 0) && ((base & 7) != RBP)) { | |
| SetModRM(0, RSP); | |
| SetSIB(scale, index, base); | |
| } else if (Utils::IsInt(8, disp)) { | |
| SetModRM(1, RSP); | |
| SetSIB(scale, index, base); | |
| SetDisp8(disp); | |
| } else { | |
| SetModRM(2, RSP); | |
| SetSIB(scale, index, base); | |
| SetDisp32(disp); | |
| } | |
| } | |
| // This addressing mode does not exist. | |
| Address(Register base, Register index, ScaleFactor scale, Register r); | |
| Address(const Address& other) : Operand(other) {} | |
| Address& operator=(const Address& other) { | |
| Operand::operator=(other); | |
| return *this; | |
| } | |
| static Address AddressRIPRelative(int32_t disp) { | |
| return Address(RIPRelativeDisp(disp)); | |
| } | |
| static Address AddressBaseImm32(Register base, int32_t disp) { | |
| return Address(base, disp, true); | |
| } | |
| // This addressing mode does not exist. | |
| static Address AddressBaseImm32(Register base, Register r); | |
| private: | |
| Address(Register base, int32_t disp, bool fixed) { | |
| ASSERT(fixed); | |
| SetModRM(2, base); | |
| if ((base & 7) == RSP) { | |
| SetSIB(TIMES_1, RSP, base); | |
| } | |
| SetDisp32(disp); | |
| } | |
| struct RIPRelativeDisp { | |
| explicit RIPRelativeDisp(int32_t disp) : disp_(disp) {} | |
| const int32_t disp_; | |
| }; | |
| explicit Address(const RIPRelativeDisp& disp) { | |
| SetModRM(0, static_cast<Register>(0x5)); | |
| SetDisp32(disp.disp_); | |
| } | |
| }; | |
| class FieldAddress : public Address { | |
| public: | |
| FieldAddress(Register base, int32_t disp) | |
| : Address(base, disp - kHeapObjectTag) {} | |
| // This addressing mode does not exist. | |
| FieldAddress(Register base, Register r); | |
| FieldAddress(Register base, Register index, ScaleFactor scale, int32_t disp) | |
| : Address(base, index, scale, disp - kHeapObjectTag) {} | |
| // This addressing mode does not exist. | |
| FieldAddress(Register base, Register index, ScaleFactor scale, Register r); | |
| FieldAddress(const FieldAddress& other) : Address(other) {} | |
| FieldAddress& operator=(const FieldAddress& other) { | |
| Address::operator=(other); | |
| return *this; | |
| } | |
| }; | |
| #if !defined(DART_COMPRESSED_POINTERS) | |
| #define OBJ(op) op##q | |
| #else | |
| #define OBJ(op) op##l | |
| #endif | |
| class Assembler : public AssemblerBase { | |
| public: | |
| explicit Assembler(ObjectPoolBuilder* object_pool_builder, | |
| intptr_t far_branch_level = 0); | |
| ~Assembler() {} | |
| /* | |
| * Emit Machine Instructions. | |
| */ | |
| void call(Register reg) { EmitUnaryL(reg, 0xFF, 2); } | |
| void call(const Address& address) { EmitUnaryL(address, 0xFF, 2); } | |
| void call(Label* label); | |
| void call(const ExternalLabel* label); | |
| void pushq(Register reg); | |
| void pushq(const Address& address) { EmitUnaryL(address, 0xFF, 6); } | |
| void pushq(const Immediate& imm); | |
| void PushImmediate(const Immediate& imm) { pushq(imm); } | |
| void PushImmediate(int64_t value) { PushImmediate(Immediate(value)); } | |
| void popq(Register reg); | |
| void popq(const Address& address) { EmitUnaryL(address, 0x8F, 0); } | |
| void setcc(Condition condition, ByteRegister dst); | |
| void EnterFullSafepoint(); | |
| void ExitFullSafepoint(); | |
| void TransitionGeneratedToNative(Register destination_address, | |
| Register new_exit_frame, | |
| Register new_exit_through_ffi, | |
| bool enter_safepoint); | |
| void TransitionNativeToGenerated(bool exit_safepoint, bool set_tag = true); | |
| void VerifyInGenerated(Register scratch); | |
| void VerifyNotInGenerated(Register scratch); | |
| // Register-register, register-address and address-register instructions. | |
| #define RR(width, name, ...) \ | |
| void name(Register dst, Register src) { \ | |
| Emit##width(dst, src, __VA_ARGS__); \ | |
| } | |
| #define RA(width, name, ...) \ | |
| void name(Register dst, const Address& src) { \ | |
| Emit##width(dst, src, __VA_ARGS__); \ | |
| } | |
| #define RAB(name, ...) \ | |
| void name(ByteRegister dst, const Address& src) { \ | |
| EmitB(dst, src, __VA_ARGS__); \ | |
| } | |
| #define AR(width, name, ...) \ | |
| void name(const Address& dst, Register src) { \ | |
| Emit##width(src, dst, __VA_ARGS__); \ | |
| } | |
| #define ARB(name, ...) \ | |
| void name(const Address& dst, ByteRegister src) { \ | |
| EmitB(src, dst, __VA_ARGS__); \ | |
| } | |
| #define REGULAR_INSTRUCTION(name, ...) \ | |
| RA(W, name##w, __VA_ARGS__) \ | |
| RA(L, name##l, __VA_ARGS__) \ | |
| RA(Q, name##q, __VA_ARGS__) \ | |
| RR(W, name##w, __VA_ARGS__) \ | |
| RR(L, name##l, __VA_ARGS__) \ | |
| RR(Q, name##q, __VA_ARGS__) | |
| REGULAR_INSTRUCTION(test, 0x85) | |
| REGULAR_INSTRUCTION(xchg, 0x87) | |
| REGULAR_INSTRUCTION(imul, 0xAF, 0x0F) | |
| REGULAR_INSTRUCTION(bsf, 0xBC, 0x0F) | |
| REGULAR_INSTRUCTION(bsr, 0xBD, 0x0F) | |
| REGULAR_INSTRUCTION(rep_bsf, 0xBC, 0x0F, 0xF3) | |
| REGULAR_INSTRUCTION(popcnt, 0xB8, 0x0F, 0xF3) | |
| REGULAR_INSTRUCTION(lzcnt, 0xBD, 0x0F, 0xF3) | |
| #undef REGULAR_INSTRUCTION | |
| RA(Q, movsxd, 0x63) | |
| RR(Q, movsxd, 0x63) | |
| ARB(movb, 0x88) | |
| AR(L, movl, 0x89) | |
| AR(Q, movq, 0x89) | |
| AR(W, movw, 0x89) | |
| RAB(movb, 0x8A) | |
| RA(L, movl, 0x8B) | |
| RA(Q, movq, 0x8B) | |
| RR(L, movl, 0x8B) | |
| RA(Q, leaq, 0x8D) | |
| RA(L, leal, 0x8D) | |
| AR(L, cmpxchgl, 0xB1, 0x0F) | |
| AR(Q, cmpxchgq, 0xB1, 0x0F) | |
| RA(L, cmpxchgl, 0xB1, 0x0F) | |
| RA(Q, cmpxchgq, 0xB1, 0x0F) | |
| RR(L, cmpxchgl, 0xB1, 0x0F) | |
| RR(Q, cmpxchgq, 0xB1, 0x0F) | |
| RA(Q, movzxb, 0xB6, 0x0F) | |
| RR(Q, movzxb, 0xB6, 0x0F) | |
| RA(Q, movzxw, 0xB7, 0x0F) | |
| RR(Q, movzxw, 0xB7, 0x0F) | |
| RA(Q, movsxb, 0xBE, 0x0F) | |
| RR(Q, movsxb, 0xBE, 0x0F) | |
| RA(Q, movsxw, 0xBF, 0x0F) | |
| RR(Q, movsxw, 0xBF, 0x0F) | |
| #define DECLARE_CMOV(name, code) \ | |
| RR(Q, cmov##name##q, 0x40 + code, 0x0F) \ | |
| RR(L, cmov##name##l, 0x40 + code, 0x0F) \ | |
| RA(Q, cmov##name##q, 0x40 + code, 0x0F) \ | |
| RA(L, cmov##name##l, 0x40 + code, 0x0F) | |
| X86_CONDITIONAL_SUFFIXES(DECLARE_CMOV) | |
| #undef DECLARE_CMOV | |
| #undef AA | |
| #undef RA | |
| #undef AR | |
| #define SIMPLE(name, ...) \ | |
| void name() { \ | |
| EmitSimple(__VA_ARGS__); \ | |
| } | |
| SIMPLE(cpuid, 0x0F, 0xA2) | |
| SIMPLE(notrack, 0x3E) | |
| SIMPLE(fcos, 0xD9, 0xFF) | |
| SIMPLE(fincstp, 0xD9, 0xF7) | |
| SIMPLE(fsin, 0xD9, 0xFE) | |
| SIMPLE(lock, 0xF0) | |
| SIMPLE(rep_movsb, 0xF3, 0xA4) | |
| SIMPLE(rep_movsw, 0xF3, 0x66, 0xA5) | |
| SIMPLE(rep_movsd, 0xF3, 0xA5) | |
| SIMPLE(rep_movsq, 0xF3, 0x48, 0xA5) | |
| #undef SIMPLE | |
| // XmmRegister operations with another register or an address. | |
| #define XX(width, name, ...) \ | |
| void name(XmmRegister dst, XmmRegister src) { \ | |
| Emit##width(dst, src, __VA_ARGS__); \ | |
| } | |
| #define XA(width, name, ...) \ | |
| void name(XmmRegister dst, const Address& src) { \ | |
| Emit##width(dst, src, __VA_ARGS__); \ | |
| } | |
| #define AX(width, name, ...) \ | |
| void name(const Address& dst, XmmRegister src) { \ | |
| Emit##width(src, dst, __VA_ARGS__); \ | |
| } | |
| // We could add movupd here, but movups does the same and is shorter. | |
| XA(L, movups, 0x10, 0x0F); | |
| XA(L, movsd, 0x10, 0x0F, 0xF2) | |
| XA(L, movss, 0x10, 0x0F, 0xF3) | |
| AX(L, movups, 0x11, 0x0F); | |
| AX(L, movsd, 0x11, 0x0F, 0xF2) | |
| AX(L, movss, 0x11, 0x0F, 0xF3) | |
| XX(L, movhlps, 0x12, 0x0F) | |
| XX(L, unpcklps, 0x14, 0x0F) | |
| XX(L, unpcklpd, 0x14, 0x0F, 0x66) | |
| XX(L, unpckhps, 0x15, 0x0F) | |
| XX(L, unpckhpd, 0x15, 0x0F, 0x66) | |
| XX(L, movlhps, 0x16, 0x0F) | |
| XX(L, movaps, 0x28, 0x0F) | |
| XX(L, comisd, 0x2F, 0x0F, 0x66) | |
| #define DECLARE_XMM(name, code) \ | |
| XX(L, name##ps, 0x50 + code, 0x0F) \ | |
| XA(L, name##ps, 0x50 + code, 0x0F) \ | |
| AX(L, name##ps, 0x50 + code, 0x0F) \ | |
| XX(L, name##pd, 0x50 + code, 0x0F, 0x66) \ | |
| XA(L, name##pd, 0x50 + code, 0x0F, 0x66) \ | |
| AX(L, name##pd, 0x50 + code, 0x0F, 0x66) \ | |
| XX(L, name##sd, 0x50 + code, 0x0F, 0xF2) \ | |
| XA(L, name##sd, 0x50 + code, 0x0F, 0xF2) \ | |
| AX(L, name##sd, 0x50 + code, 0x0F, 0xF2) \ | |
| XX(L, name##ss, 0x50 + code, 0x0F, 0xF3) \ | |
| XA(L, name##ss, 0x50 + code, 0x0F, 0xF3) \ | |
| AX(L, name##ss, 0x50 + code, 0x0F, 0xF3) | |
| XMM_ALU_CODES(DECLARE_XMM) | |
| #undef DECLARE_XMM | |
| XX(L, cvtps2pd, 0x5A, 0x0F) | |
| XX(L, cvtpd2ps, 0x5A, 0x0F, 0x66) | |
| XX(L, cvtsd2ss, 0x5A, 0x0F, 0xF2) | |
| XX(L, cvtss2sd, 0x5A, 0x0F, 0xF3) | |
| XX(L, pxor, 0xEF, 0x0F, 0x66) | |
| XX(L, pcmpeqd, 0x76, 0x0F, 0x66) | |
| XX(L, subpl, 0xFA, 0x0F, 0x66) | |
| XX(L, addpl, 0xFE, 0x0F, 0x66) | |
| XX(L, pslld, 0xF2, 0x0F, 0x66) | |
| XX(L, psrad, 0xE2, 0x0F, 0x66) | |
| XX(L, psrld, 0xD2, 0x0F, 0x66) | |
| #undef XX | |
| #undef AX | |
| #undef XA | |
| #define DECLARE_CMPPS(name, code) \ | |
| void cmpps##name(XmmRegister dst, XmmRegister src) { \ | |
| EmitL(dst, src, 0xC2, 0x0F); \ | |
| AssemblerBuffer::EnsureCapacity ensured(&buffer_); \ | |
| EmitUint8(code); \ | |
| } | |
| XMM_CONDITIONAL_CODES(DECLARE_CMPPS) | |
| #undef DECLARE_CMPPS | |
| #define DECLARE_SIMPLE(name, opcode) \ | |
| void name() { \ | |
| EmitSimple(opcode); \ | |
| } | |
| X86_ZERO_OPERAND_1_BYTE_INSTRUCTIONS(DECLARE_SIMPLE) | |
| #undef DECLARE_SIMPLE | |
| void movl(Register dst, const Immediate& imm); | |
| void movl(const Address& dst, const Immediate& imm); | |
| void movb(const Address& dst, const Immediate& imm); | |
| void movw(Register dst, const Address& src); | |
| void movw(const Address& dst, const Immediate& imm); | |
| void movq(Register dst, const Immediate& imm); | |
| void movq(const Address& dst, const Immediate& imm); | |
| // Destination and source are reversed for some reason. | |
| void movq(Register dst, XmmRegister src) { | |
| EmitQ(src, dst, 0x7E, 0x0F, 0x66); | |
| } | |
| void movl(Register dst, XmmRegister src) { | |
| EmitL(src, dst, 0x7E, 0x0F, 0x66); | |
| } | |
| void movss(XmmRegister dst, XmmRegister src) { | |
| EmitL(src, dst, 0x11, 0x0F, 0xF3); | |
| } | |
| void movsd(XmmRegister dst, XmmRegister src) { | |
| EmitL(src, dst, 0x11, 0x0F, 0xF2); | |
| } | |
| // Use the reversed operand order and the 0x89 bytecode instead of the | |
| // obvious 0x88 encoding for this some, because it is expected by gdb64 older | |
| // than 7.3.1-gg5 when disassembling a function's prologue (movq rbp, rsp) | |
| // for proper unwinding of Dart frames (use --generate_gdb_symbols and -O0). | |
| void movq(Register dst, Register src) { EmitQ(src, dst, 0x89); } | |
| void movq(XmmRegister dst, Register src) { | |
| EmitQ(dst, src, 0x6E, 0x0F, 0x66); | |
| } | |
| void movd(XmmRegister dst, Register src) { | |
| EmitL(dst, src, 0x6E, 0x0F, 0x66); | |
| } | |
| void cvtsi2sdq(XmmRegister dst, Register src) { | |
| EmitQ(dst, src, 0x2A, 0x0F, 0xF2); | |
| } | |
| void cvtsi2sdl(XmmRegister dst, Register src) { | |
| EmitL(dst, src, 0x2A, 0x0F, 0xF2); | |
| } | |
| void cvttsd2siq(Register dst, XmmRegister src) { | |
| EmitQ(dst, src, 0x2C, 0x0F, 0xF2); | |
| } | |
| void cvttsd2sil(Register dst, XmmRegister src) { | |
| EmitL(dst, src, 0x2C, 0x0F, 0xF2); | |
| } | |
| void movmskpd(Register dst, XmmRegister src) { | |
| EmitL(dst, src, 0x50, 0x0F, 0x66); | |
| } | |
| void movmskps(Register dst, XmmRegister src) { EmitL(dst, src, 0x50, 0x0F); } | |
| void pmovmskb(Register dst, XmmRegister src) { | |
| EmitL(dst, src, 0xD7, 0x0F, 0x66); | |
| } | |
| void btl(Register dst, Register src) { EmitL(src, dst, 0xA3, 0x0F); } | |
| void btq(Register dst, Register src) { EmitQ(src, dst, 0xA3, 0x0F); } | |
| void notps(XmmRegister dst, XmmRegister src); | |
| void negateps(XmmRegister dst, XmmRegister src); | |
| void absps(XmmRegister dst, XmmRegister src); | |
| void zerowps(XmmRegister dst, XmmRegister src); | |
| void set1ps(XmmRegister dst, Register tmp, const Immediate& imm); | |
| void shufps(XmmRegister dst, XmmRegister src, const Immediate& mask); | |
| void negatepd(XmmRegister dst, XmmRegister src); | |
| void abspd(XmmRegister dst, XmmRegister src); | |
| void shufpd(XmmRegister dst, XmmRegister src, const Immediate& mask); | |
| void pshufd(XmmRegister dst, XmmRegister src, const Immediate& mask); | |
| enum RoundingMode { | |
| kRoundToNearest = 0x0, | |
| kRoundDown = 0x1, | |
| kRoundUp = 0x2, | |
| kRoundToZero = 0x3 | |
| }; | |
| void roundsd(XmmRegister dst, XmmRegister src, RoundingMode mode); | |
| void ptest(XmmRegister dst, XmmRegister src); | |
| void CompareImmediate(Register reg, | |
| const Immediate& imm, | |
| OperandSize width = kEightBytes); | |
| void CompareImmediate(const Address& address, | |
| const Immediate& imm, | |
| OperandSize width = kEightBytes); | |
| void CompareImmediate(Register reg, | |
| int64_t immediate, | |
| OperandSize width = kEightBytes) override { | |
| return CompareImmediate(reg, Immediate(immediate), width); | |
| } | |
| void testl(Register reg, const Immediate& imm) { | |
| testq(reg, Immediate(imm.value() & 0xFFFFFFFF)); | |
| } | |
| void testb(const Address& address, const Immediate& imm); | |
| void testb(const Address& address, Register reg); | |
| void testq(Register reg, const Immediate& imm); | |
| void TestImmediate(Register dst, | |
| const Immediate& imm, | |
| OperandSize width = kEightBytes); | |
| void AndImmediate(Register dst, | |
| Register src, | |
| const Immediate& imm, | |
| OperandSize sz = kWordBytes); | |
| void AndImmediate(Register dst, | |
| Register src, | |
| int64_t value, | |
| OperandSize sz = kWordBytes) override { | |
| AndImmediate(dst, src, Immediate(value), sz); | |
| } | |
| void AndImmediate(Register reg, | |
| const Immediate& imm, | |
| OperandSize sz = kWordBytes) { | |
| AndImmediate(reg, reg, imm, sz); | |
| } | |
| void AndImmediate(Register reg, | |
| int64_t value, | |
| OperandSize sz = kWordBytes) override { | |
| AndImmediate(reg, reg, value, sz); | |
| } | |
| void AndRegisters(Register dst, | |
| Register src1, | |
| Register src2 = kNoRegister) override; | |
| void OrImmediate(Register dst, const Immediate& imm); | |
| void OrImmediate(Register dst, int64_t value) { | |
| OrImmediate(dst, Immediate(value)); | |
| } | |
| void XorImmediate(Register dst, const Immediate& imm); | |
| void LslImmediate(Register dst, | |
| Register src, | |
| int32_t shift, | |
| OperandSize sz = kWordBytes) override; | |
| void LslImmediate(Register reg, | |
| int32_t shift, | |
| OperandSize sz = kWordBytes) override { | |
| LslImmediate(reg, reg, shift, sz); | |
| } | |
| void LslRegister(Register dst, Register shift) override; | |
| void LsrImmediate(Register dst, int32_t shift) override { | |
| shrq(dst, Immediate(shift)); | |
| } | |
| void shldq(Register dst, Register src, Register shifter) { | |
| ASSERT(shifter == RCX); | |
| EmitQ(src, dst, 0xA5, 0x0F); | |
| } | |
| void shrdq(Register dst, Register src, Register shifter) { | |
| ASSERT(shifter == RCX); | |
| EmitQ(src, dst, 0xAD, 0x0F); | |
| } | |
| #define DECLARE_ALU(op, c) \ | |
| void op##w(Register dst, Register src) { \ | |
| EmitW(dst, src, c * 8 + 3); \ | |
| } \ | |
| void op##l(Register dst, Register src) { \ | |
| EmitL(dst, src, c * 8 + 3); \ | |
| } \ | |
| void op##q(Register dst, Register src) { \ | |
| EmitQ(dst, src, c * 8 + 3); \ | |
| } \ | |
| void op##w(Register dst, const Address& src) { \ | |
| EmitW(dst, src, c * 8 + 3); \ | |
| } \ | |
| void op##l(Register dst, const Address& src) { \ | |
| EmitL(dst, src, c * 8 + 3); \ | |
| } \ | |
| void op##q(Register dst, const Address& src) { \ | |
| EmitQ(dst, src, c * 8 + 3); \ | |
| } \ | |
| void op##w(const Address& dst, Register src) { \ | |
| EmitW(src, dst, c * 8 + 1); \ | |
| } \ | |
| void op##l(const Address& dst, Register src) { \ | |
| EmitL(src, dst, c * 8 + 1); \ | |
| } \ | |
| void op##q(const Address& dst, Register src) { \ | |
| EmitQ(src, dst, c * 8 + 1); \ | |
| } \ | |
| void op##l(Register dst, const Immediate& imm) { \ | |
| AluL(c, dst, imm); \ | |
| } \ | |
| void op##q(Register dst, const Immediate& imm) { \ | |
| AluQ(c, c * 8 + 3, dst, imm); \ | |
| } \ | |
| void op##b(const Address& dst, const Immediate& imm) { \ | |
| AluB(c, dst, imm); \ | |
| } \ | |
| void op##w(const Address& dst, const Immediate& imm) { \ | |
| AluW(c, dst, imm); \ | |
| } \ | |
| void op##l(const Address& dst, const Immediate& imm) { \ | |
| AluL(c, dst, imm); \ | |
| } \ | |
| void op##q(const Address& dst, const Immediate& imm) { \ | |
| AluQ(c, c * 8 + 3, dst, imm); \ | |
| } | |
| X86_ALU_CODES(DECLARE_ALU) | |
| #undef DECLARE_ALU | |
| #undef ALU_OPS | |
| void cqo(); | |
| #define REGULAR_UNARY(name, opcode, modrm) \ | |
| void name##q(Register reg) { \ | |
| EmitUnaryQ(reg, opcode, modrm); \ | |
| } \ | |
| void name##l(Register reg) { \ | |
| EmitUnaryL(reg, opcode, modrm); \ | |
| } \ | |
| void name##q(const Address& address) { \ | |
| EmitUnaryQ(address, opcode, modrm); \ | |
| } \ | |
| void name##l(const Address& address) { \ | |
| EmitUnaryL(address, opcode, modrm); \ | |
| } | |
| REGULAR_UNARY(not, 0xF7, 2) | |
| REGULAR_UNARY(neg, 0xF7, 3) | |
| REGULAR_UNARY(mul, 0xF7, 4) | |
| REGULAR_UNARY(imul, 0xF7, 5) | |
| REGULAR_UNARY(div, 0xF7, 6) | |
| REGULAR_UNARY(idiv, 0xF7, 7) | |
| REGULAR_UNARY(inc, 0xFF, 0) | |
| REGULAR_UNARY(dec, 0xFF, 1) | |
| #undef REGULAR_UNARY | |
| void imull(Register reg, const Immediate& imm); | |
| void imulq(Register dst, const Immediate& imm); | |
| void MulImmediate(Register reg, | |
| const Immediate& imm, | |
| OperandSize width = kEightBytes); | |
| void MulImmediate(Register reg, | |
| int64_t imm, | |
| OperandSize width = kEightBytes) override { | |
| MulImmediate(reg, Immediate(imm), width); | |
| } | |
| void shll(Register reg, const Immediate& imm); | |
| void shll(Register operand, Register shifter); | |
| void shrl(Register reg, const Immediate& imm); | |
| void shrl(Register operand, Register shifter); | |
| void sarl(Register reg, const Immediate& imm); | |
| void sarl(Register operand, Register shifter); | |
| void shldl(Register dst, Register src, const Immediate& imm); | |
| void shlq(Register reg, const Immediate& imm); | |
| void shlq(Register operand, Register shifter); | |
| void shrq(Register reg, const Immediate& imm); | |
| void shrq(Register operand, Register shifter); | |
| void sarq(Register reg, const Immediate& imm); | |
| void sarq(Register operand, Register shifter); | |
| void shldq(Register dst, Register src, const Immediate& imm); | |
| void btq(Register base, int bit); | |
| void btrq(const Address& base, int bit); | |
| void btsq(const Address& base, int bit); | |
| void enter(const Immediate& imm); | |
| void fldl(const Address& src); | |
| void fstpl(const Address& dst); | |
| void ffree(intptr_t value); | |
| // 'size' indicates size in bytes and must be in the range 1..8. | |
| void nop(int size = 1); | |
| void endbr64(); | |
| void j(Condition condition, Label* label, JumpDistance distance = kFarJump); | |
| void jmp(Register reg) { EmitUnaryL(reg, 0xFF, 4); } | |
| void jmp(const Address& address) { EmitUnaryL(address, 0xFF, 4); } | |
| void jmp(Label* label, JumpDistance distance = kFarJump); | |
| void jmp(const ExternalLabel* label); | |
| void jmp(const Code& code); | |
| /// Moves an XMM register's content to a 64-bit register. | |
| void MoveFpuRegisterToRegister(Register dst, FpuRegister src) { | |
| movq(dst, src); | |
| } | |
| // Issue memory to memory move through a TMP register. | |
| // TODO(koda): Assert that these are not used for heap objects. | |
| void MoveMemoryToMemory(const Address& dst, const Address& src) { | |
| movq(TMP, src); | |
| movq(dst, TMP); | |
| } | |
| void Exchange(Register reg, const Address& mem) { | |
| movq(TMP, mem); | |
| movq(mem, reg); | |
| movq(reg, TMP); | |
| } | |
| void Exchange(const Address& mem1, const Address& mem2) { | |
| movq(TMP, mem1); | |
| xorq(TMP, mem2); | |
| xorq(mem1, TMP); | |
| xorq(mem2, TMP); | |
| } | |
| // Methods for High-level operations and implemented on all architectures. | |
| void Ret() { ret(); } | |
| // Sets the return address to [value] as if there was a call. | |
| // On X64 pushes [value]. | |
| void SetReturnAddress(Register value) { PushRegister(value); } | |
| void CompareRegisters(Register a, Register b); | |
| void CompareObjectRegisters(Register a, Register b) { OBJ(cmp)(a, b); } | |
| void BranchIf(Condition condition, | |
| Label* label, | |
| JumpDistance distance = kFarJump) { | |
| j(condition, label, distance); | |
| } | |
| void BranchIfZero(Register src, | |
| Label* label, | |
| JumpDistance distance = kFarJump) { | |
| cmpq(src, Immediate(0)); | |
| j(ZERO, label, distance); | |
| } | |
| void BranchIfBit(Register rn, | |
| intptr_t bit_number, | |
| Condition condition, | |
| Label* label, | |
| JumpDistance distance = kFarJump) { | |
| testq(rn, Immediate(1 << bit_number)); | |
| j(condition, label, distance); | |
| } | |
| void ExtendValue(Register dst, Register src, OperandSize sz) override; | |
| void PushRegister(Register r); | |
| void PopRegister(Register r); | |
| void PushRegisterPair(Register r0, Register r1) { | |
| PushRegister(r1); | |
| PushRegister(r0); | |
| } | |
| void PopRegisterPair(Register r0, Register r1) { | |
| PopRegister(r0); | |
| PopRegister(r1); | |
| } | |
| void PushValueAtOffset(Register base, int32_t offset) { | |
| pushq(Address(base, offset)); | |
| } | |
| // Methods for adding/subtracting an immediate value that may be loaded from | |
| // the constant pool. | |
| // TODO(koda): Assert that these are not used for heap objects. | |
| void AddImmediate(Register reg, | |
| const Immediate& imm, | |
| OperandSize width = kEightBytes); | |
| void AddImmediate(Register reg, | |
| int64_t value, | |
| OperandSize width = kEightBytes) { | |
| AddImmediate(reg, Immediate(value), width); | |
| } | |
| void AddRegisters(Register dest, Register src) { addq(dest, src); } | |
| void AddScaled(Register dest, | |
| Register base, | |
| Register index, | |
| ScaleFactor scale, | |
| int32_t disp) override { | |
| if (base == kNoRegister) { | |
| leaq(dest, Address(index, scale, disp)); | |
| } else { | |
| leaq(dest, Address(base, index, scale, disp)); | |
| } | |
| } | |
| void AddImmediate(Register dest, Register src, int64_t value); | |
| void AddImmediate(const Address& address, const Immediate& imm); | |
| void SubImmediate(Register reg, | |
| const Immediate& imm, | |
| OperandSize width = kEightBytes); | |
| void SubImmediate(const Address& address, const Immediate& imm); | |
| void SubRegisters(Register dest, Register src) { subq(dest, src); } | |
| void Drop(intptr_t stack_elements, Register tmp = TMP); | |
| bool constant_pool_allowed() const { return constant_pool_allowed_; } | |
| void set_constant_pool_allowed(bool b) { constant_pool_allowed_ = b; } | |
| // Unlike movq this can affect the flags or use the constant pool. | |
| void LoadImmediate(Register reg, const Immediate& imm); | |
| void LoadImmediate(Register reg, int64_t immediate) override { | |
| LoadImmediate(reg, Immediate(immediate)); | |
| } | |
| void LoadSImmediate(FpuRegister dst, float immediate); | |
| void LoadDImmediate(FpuRegister dst, double immediate); | |
| void LoadQImmediate(FpuRegister dst, simd128_value_t immediate); | |
| // Sets register to zero. | |
| // Affects flags (sets zero flag, clears rest). | |
| void ClearRegister(Register reg) { xorl(reg, reg); } | |
| // Sets XMM register to zero. | |
| // Affects flags (sets zero flag, clears rest). | |
| void ClearFpuRegister(FpuRegister reg) { xorps(reg, reg); } | |
| void LoadIsolate(Register dst); | |
| void LoadIsolateGroup(Register dst); | |
| void LoadDispatchTable(Register dst); | |
| void LoadObject(Register dst, const Object& obj); | |
| void LoadUniqueObject( | |
| Register dst, | |
| const Object& obj, | |
| ObjectPoolBuilderEntry::SnapshotBehavior snapshot_behavior = | |
| ObjectPoolBuilderEntry::kSnapshotable); | |
| void LoadNativeEntry(Register dst, | |
| const ExternalLabel* label, | |
| ObjectPoolBuilderEntry::Patchability patchable); | |
| void JmpPatchable(const Code& code, Register pp); | |
| void Jmp(const Code& code, Register pp = PP); | |
| void J(Condition condition, const Code& code, Register pp); | |
| void CallPatchable( | |
| const Code& code, | |
| CodeEntryKind entry_kind = CodeEntryKind::kNormal, | |
| ObjectPoolBuilderEntry::SnapshotBehavior snapshot_behavior = | |
| ObjectPoolBuilderEntry::kSnapshotable); | |
| void Call(const Code& stub_entry, | |
| ObjectPoolBuilderEntry::SnapshotBehavior snapshot_behavior = | |
| ObjectPoolBuilderEntry::kSnapshotable); | |
| // Emit a call that shares its object pool entries with other calls | |
| // that have the same equivalence marker. | |
| void CallWithEquivalence(const Code& code, | |
| const Object& equivalence, | |
| CodeEntryKind entry_kind = CodeEntryKind::kNormal); | |
| void Call(Address target) { call(target); } | |
| void InitializeHeader(Register tags, Register object) { | |
| movq(FieldAddress(object, target::Object::tags_offset()), tags); | |
| // No fence: all stores are ordered on x64. | |
| } | |
| void InitializeHeaderUntagged(Register tags, Register object) { | |
| movq(Address(object, target::Object::tags_offset()), tags); | |
| // No fence: all stores are ordered on x64. | |
| } | |
| void InitializeHeader(Immediate tags, Register object) { | |
| movq(FieldAddress(object, target::Object::tags_offset()), tags); | |
| // No fence: all stores are ordered on x64. | |
| } | |
| // Unaware of write barrier (use StoreInto* methods for storing to objects). | |
| // TODO(koda): Add StackAddress/HeapAddress types to prevent misuse. | |
| void StoreObject(const Address& dst, | |
| const Object& obj, | |
| OperandSize size = kWordBytes); | |
| void PushObject(const Object& object); | |
| void CompareObject(Register reg, const Object& object); | |
| #if defined(DART_COMPRESSED_POINTERS) | |
| void LoadCompressed(Register dest, const Address& slot) override; | |
| #endif | |
| void StoreBarrier(Register object, // Object we are storing into. | |
| Register value, // Value we are storing. | |
| CanBeSmi can_be_smi, | |
| Register scratch) override; | |
| void ArrayStoreBarrier(Register object, // Object we are storing into. | |
| Register slot, // Slot into which we are storing. | |
| Register value, // Value we are storing. | |
| CanBeSmi can_be_smi, | |
| Register scratch) override; | |
| void VerifyStoreNeedsNoWriteBarrier(Register object, Register value) override; | |
| void StoreObjectIntoObjectNoBarrier( | |
| Register object, | |
| const Address& dest, | |
| const Object& value, | |
| MemoryOrder memory_order = kRelaxedNonAtomic, | |
| OperandSize size = kWordBytes) override; | |
| // Stores a non-tagged value into a heap object. | |
| void StoreInternalPointer(Register object, | |
| const Address& dest, | |
| Register value); | |
| // Stores a Smi value into a heap object field that always contains a Smi. | |
| void StoreIntoSmiField(const Address& dest, Register value); | |
| void ZeroInitSmiField(const Address& dest); | |
| void ZeroInitCompressedSmiField(const Address& dest); | |
| // Increments a Smi field. Leaves flags in same state as an 'addq'. | |
| void IncrementCompressedSmiField(const Address& dest, int64_t increment); | |
| void DoubleNegate(XmmRegister dst, XmmRegister src); | |
| void DoubleAbs(XmmRegister dst, XmmRegister src); | |
| void LockCmpxchgq(const Address& address, Register reg) { | |
| lock(); | |
| cmpxchgq(address, reg); | |
| } | |
| void LockCmpxchgl(const Address& address, Register reg) { | |
| lock(); | |
| cmpxchgl(address, reg); | |
| } | |
| void PushRegisters(const RegisterSet& registers); | |
| void PopRegisters(const RegisterSet& registers); | |
| void PushRegistersAligned(const RegisterSet& registers, intptr_t space); | |
| void PopRegistersAligned(const RegisterSet& registers, intptr_t space); | |
| void PushRegistersInOrder(std::initializer_list<Register> regs); | |
| void CheckCodePointer(); | |
| void EnterFrame(intptr_t frame_space); | |
| void LeaveFrame(); | |
| void ReserveAlignedFrameSpace(intptr_t frame_space); | |
| // In debug mode, generates code to verify that: | |
| // FP + kExitLinkSlotFromFp == SP | |
| // | |
| // Triggers breakpoint otherwise. | |
| // Clobbers RAX. | |
| void EmitEntryFrameVerification(); | |
| // For non-leaf runtime calls. For leaf runtime calls, use LeafRuntimeScope, | |
| void CallRuntime(const RuntimeEntry& entry, | |
| intptr_t argument_count, | |
| bool tsan_enter_exit = true); | |
| // Call runtime function. Reserves shadow space on the stack before calling | |
| // if platform ABI requires that. | |
| void CallCFunction(Register reg, bool restore_rsp = false); | |
| void CallCFunction(Address address, bool restore_rsp = false); | |
| void ExtractBitField(Register dst, | |
| Register src, | |
| intptr_t low_bit, | |
| intptr_t width) override; | |
| void ExtractClassIdFromTags(Register result, Register tags); | |
| void ExtractInstanceSizeFromTags(Register result, Register tags); | |
| void RangeCheck(Register value, | |
| Register temp, | |
| intptr_t low, | |
| intptr_t high, | |
| RangeCheckCondition condition, | |
| Label* target) override; | |
| // Loading and comparing classes of objects. | |
| void LoadClassId(Register result, Register object); | |
| void LoadClassById(Register result, Register class_id); | |
| void CompareClassId(Register object, | |
| intptr_t class_id, | |
| Register scratch = kNoRegister); | |
| void LoadClassIdMayBeSmi(Register result, Register object); | |
| void LoadTaggedClassIdMayBeSmi(Register result, Register object); | |
| void EnsureHasClassIdInDEBUG(intptr_t cid, | |
| Register src, | |
| Register scratch, | |
| bool can_be_null = false) override; | |
| #if defined(DART_COMPRESSED_POINTERS) | |
| void ExtendNonNegativeSmi(Register dst) override { | |
| // Zero-extends and is a smaller instruction to output than sign | |
| // extension (movsxd). | |
| orl(dst, dst); | |
| } | |
| #endif | |
| // CheckClassIs fused with optimistic SmiUntag. | |
| // Value in the register object is untagged optimistically. | |
| void SmiUntagOrCheckClass(Register object, intptr_t class_id, Label* smi); | |
| // Misc. functionality. | |
| void SmiTag(Register reg) override { OBJ(add)(reg, reg); } | |
| void SmiUntag(Register reg) { OBJ(sar)(reg, Immediate(kSmiTagSize)); } | |
| void SmiUntag(Register dst, Register src) { | |
| if (dst != src) { | |
| OBJ(mov)(dst, src); | |
| } | |
| OBJ(sar)(dst, Immediate(kSmiTagSize)); | |
| } | |
| void SmiUntagAndSignExtend(Register reg) { | |
| #if !defined(DART_COMPRESSED_POINTERS) | |
| sarq(reg, Immediate(kSmiTagSize)); | |
| #else | |
| // This is shorter than | |
| // shlq reg, 32 | |
| // sraq reg, 33 | |
| sarl(reg, Immediate(kSmiTagSize)); | |
| movsxd(reg, reg); | |
| #endif | |
| } | |
| void SmiUntagAndSignExtend(Register dst, Register src) { | |
| #if !defined(DART_COMPRESSED_POINTERS) | |
| if (dst != src) { | |
| movq(dst, src); | |
| } | |
| sarq(dst, Immediate(kSmiTagSize)); | |
| #else | |
| movsxd(dst, src); | |
| sarq(dst, Immediate(kSmiTagSize)); | |
| #endif | |
| } | |
| // Truncates upper bits. | |
| void LoadInt32FromBoxOrSmi(Register result, Register value) override; | |
| void LoadInt64FromBoxOrSmi(Register result, Register value) override; | |
| void BranchIfNotSmi(Register reg, | |
| Label* label, | |
| JumpDistance distance = kFarJump) { | |
| testq(reg, Immediate(kSmiTagMask)); | |
| j(NOT_ZERO, label, distance); | |
| } | |
| void BranchIfSmi(Register reg, | |
| Label* label, | |
| JumpDistance distance = kFarJump) override { | |
| testq(reg, Immediate(kSmiTagMask)); | |
| j(ZERO, label, distance); | |
| } | |
| void ArithmeticShiftRightImmediate(Register dst, | |
| Register src, | |
| int32_t shift, | |
| OperandSize sz = kEightBytes) override; | |
| void ArithmeticShiftRightImmediate(Register reg, | |
| int32_t shift, | |
| OperandSize sz = kEightBytes) override { | |
| ArithmeticShiftRightImmediate(reg, reg, shift, sz); | |
| } | |
| void CompareWords(Register reg1, | |
| Register reg2, | |
| intptr_t offset, | |
| Register count, | |
| Register temp, | |
| Label* equals) override; | |
| void Align(int alignment, intptr_t offset); | |
| void Bind(Label* label) override; | |
| // Unconditional jump to a given label. | |
| void Jump(Label* label, JumpDistance distance = kFarJump) { | |
| jmp(label, distance); | |
| } | |
| // Unconditional jump to a given address in register. | |
| void Jump(Register target) { jmp(target); } | |
| // Unconditional jump to a given address in memory. | |
| void Jump(const Address& address) { jmp(address); } | |
| // Arch-specific LoadFromOffset to choose the right operation for [sz]. | |
| void Load(Register dst, | |
| const Address& address, | |
| OperandSize sz = kEightBytes) override; | |
| void LoadIndexedPayload(Register dst, | |
| Register base, | |
| int32_t payload_offset, | |
| Register index, | |
| ScaleFactor scale, | |
| OperandSize sz = kEightBytes) override { | |
| Load(dst, FieldAddress(base, index, scale, payload_offset), sz); | |
| } | |
| #if defined(DART_COMPRESSED_POINTERS) | |
| void LoadIndexedCompressed(Register dst, | |
| Register base, | |
| int32_t offset, | |
| Register index) override { | |
| LoadCompressed( | |
| dst, FieldAddress(base, index, TIMES_COMPRESSED_WORD_SIZE, offset)); | |
| } | |
| #endif | |
| void Store(Register src, | |
| const Address& address, | |
| OperandSize sz = kEightBytes) override; | |
| void StoreZero(const Address& address, Register temp = kNoRegister) { | |
| movq(address, Immediate(0)); | |
| } | |
| void LoadFromStack(Register dst, intptr_t depth); | |
| void StoreToStack(Register src, intptr_t depth); | |
| void CompareToStack(Register src, intptr_t depth); | |
| void LoadMemoryValue(Register dst, Register base, int32_t offset) { | |
| movq(dst, Address(base, offset)); | |
| } | |
| void LoadCompressedMemoryValue(Register dst, Register base, int32_t offset) { | |
| OBJ(mov)(dst, Address(base, offset)); | |
| } | |
| void StoreMemoryValue(Register src, Register base, int32_t offset) { | |
| movq(Address(base, offset), src); | |
| } | |
| void LoadUnboxedSimd128(FpuRegister dst, Register base, int32_t offset) { | |
| movups(dst, Address(base, offset)); | |
| } | |
| void StoreUnboxedSimd128(FpuRegister dst, Register base, int32_t offset) { | |
| movups(Address(base, offset), dst); | |
| } | |
| void MoveUnboxedSimd128(FpuRegister dst, FpuRegister src) { | |
| if (src != dst) { | |
| movaps(dst, src); | |
| } | |
| } | |
| void LoadUnboxedSingle(FpuRegister dst, Register base, int32_t offset) { | |
| movss(dst, Address(base, offset)); | |
| } | |
| void LoadUnboxedDouble(FpuRegister dst, Register base, int32_t offset) { | |
| movsd(dst, Address(base, offset)); | |
| } | |
| void StoreUnboxedDouble(FpuRegister src, Register base, int32_t offset) { | |
| movsd(Address(base, offset), src); | |
| } | |
| void MoveUnboxedDouble(FpuRegister dst, FpuRegister src) { | |
| if (src != dst) { | |
| movaps(dst, src); | |
| } | |
| } | |
| void TsanLoadAcquire(Register dst, Address addr, OperandSize size); | |
| void TsanStoreRelease(Register src, Address addr, OperandSize size); | |
| void TsanFuncEntry(bool preserve_registers = true); | |
| void TsanFuncExit(bool preserve_registers = true); | |
| void LoadAcquire(Register dst, | |
| const Address& address, | |
| OperandSize size = kEightBytes) override { | |
| if (FLAG_target_thread_sanitizer) { | |
| TsanLoadAcquire(dst, address, size); | |
| } else { | |
| // On intel loads have load-acquire behavior (i.e. loads are not | |
| // re-ordered with other loads). | |
| Load(dst, address, size); | |
| } | |
| } | |
| #if defined(DART_COMPRESSED_POINTERS) | |
| void LoadAcquireCompressed(Register dst, const Address& address) override { | |
| LoadAcquire(dst, address, kUnsignedFourBytes); | |
| addq(dst, Address(THR, target::Thread::heap_base_offset())); | |
| } | |
| #endif | |
| void StoreRelease(Register src, | |
| const Address& address, | |
| OperandSize size = kWordBytes) override { | |
| if (FLAG_target_thread_sanitizer) { | |
| TsanStoreRelease(src, address, size); | |
| } else { | |
| // On intel stores have store-release behavior (i.e. stores are not | |
| // re-ordered with other stores). | |
| Store(src, address, size); | |
| } | |
| } | |
| void CompareWithMemoryValue(Register value, | |
| Address address, | |
| OperandSize size = kEightBytes) override { | |
| ASSERT(size == kEightBytes || size == kFourBytes); | |
| if (size == kFourBytes) { | |
| cmpl(value, address); | |
| } else { | |
| cmpq(value, address); | |
| } | |
| } | |
| void RestoreCodePointer(); | |
| void LoadPoolPointer(Register pp = PP); | |
| // Set up a Dart frame on entry with a frame pointer and PC information to | |
| // enable easy access to the RawInstruction object of code corresponding | |
| // to this frame. | |
| // The dart frame layout is as follows: | |
| // .... | |
| // locals space <=== RSP | |
| // saved PP | |
| // code object (used to derive the RawInstruction Object of the dart code) | |
| // saved RBP <=== RBP | |
| // ret PC | |
| // ..... | |
| // This code sets this up with the sequence: | |
| // pushq rbp | |
| // movq rbp, rsp | |
| // call L | |
| // L: <code to adjust saved pc if there is any intrinsification code> | |
| // ... | |
| // pushq r15 | |
| // ..... | |
| void EnterDartFrame(intptr_t frame_size, Register new_pp = kNoRegister); | |
| void LeaveDartFrame(); | |
| // Set up a Dart frame for a function compiled for on-stack replacement. | |
| // The frame layout is a normal Dart frame, but the frame is partially set | |
| // up on entry (it is the frame of the unoptimized code). | |
| void EnterOsrFrame(intptr_t extra_size); | |
| // Set up a stub frame so that the stack traversal code can easily identify | |
| // a stub frame. | |
| // The stub frame layout is as follows: | |
| // .... <=== RSP | |
| // pc (used to derive the RawInstruction Object of the stub) | |
| // saved RBP <=== RBP | |
| // ret PC | |
| // ..... | |
| // This code sets this up with the sequence: | |
| // pushq rbp | |
| // movq rbp, rsp | |
| // pushq immediate(0) | |
| // ..... | |
| void EnterStubFrame(); | |
| void LeaveStubFrame(); | |
| // Set up a frame for calling a C function. | |
| // Automatically save the pinned registers in Dart which are not callee- | |
| // saved in the native calling convention. | |
| // Use together with CallCFunction. | |
| void EnterCFrame(intptr_t frame_space); | |
| void LeaveCFrame(); | |
| void MonomorphicCheckedEntryJIT(); | |
| void MonomorphicCheckedEntryAOT(); | |
| void BranchOnMonomorphicCheckedEntryJIT(Label* label); | |
| void CombineHashes(Register dst, Register other) override; | |
| void FinalizeHashForSize(intptr_t bit_size, | |
| Register dst, | |
| Register scratch = TMP) override; | |
| // If allocation tracing for |cid| is enabled, will jump to |trace| label, | |
| // which will allocate in the runtime where tracing occurs. | |
| void MaybeTraceAllocation(intptr_t cid, | |
| Label* trace, | |
| Register temp_reg = kNoRegister, | |
| JumpDistance distance = JumpDistance::kFarJump); | |
| void MaybeTraceAllocation(Register cid, | |
| Label* trace, | |
| Register temp_reg = kNoRegister, | |
| JumpDistance distance = JumpDistance::kFarJump); | |
| void TryAllocateObject(intptr_t cid, | |
| intptr_t instance_size, | |
| Label* failure, | |
| JumpDistance distance, | |
| Register instance_reg, | |
| Register temp) override; | |
| void TryAllocateArray(intptr_t cid, | |
| intptr_t instance_size, | |
| Label* failure, | |
| JumpDistance distance, | |
| Register instance, | |
| Register end_address, | |
| Register temp); | |
| // Copy [size] bytes from [src] address to [dst] address. | |
| // [size] should be a multiple of word size. | |
| // Clobbers [src], [dst], [size] and [temp] registers. | |
| // X64 requires fixed registers for memory copying: | |
| // [src] = RSI, [dst] = RDI, [size] = RCX. | |
| void CopyMemoryWords(Register src, | |
| Register dst, | |
| Register size, | |
| Register temp = kNoRegister); | |
| // This emits an PC-relative call of the form "callq *[rip+<offset>]". The | |
| // offset is not yet known and needs therefore relocation to the right place | |
| // before the code can be used. | |
| // | |
| // The necessary information for the "linker" (i.e. the relocation | |
| // information) is stored in [UntaggedCode::static_calls_target_table_]: an | |
| // entry of the form | |
| // | |
| // (Code::kPcRelativeCall & pc_offset, <target-code>, <target-function>) | |
| // | |
| // will be used during relocation to fix the offset. | |
| // | |
| // The provided [offset_into_target] will be added to calculate the final | |
| // destination. It can be used e.g. for calling into the middle of a | |
| // function. | |
| void GenerateUnRelocatedPcRelativeCall(intptr_t offset_into_target = 0); | |
| // This emits an PC-relative tail call of the form "jmp *[rip+<offset>]". | |
| // | |
| // See also above for the pc-relative call. | |
| void GenerateUnRelocatedPcRelativeTailCall(intptr_t offset_into_target = 0); | |
| // Debugging and bringup support. | |
| void Breakpoint() override { int3(); } | |
| void StoreStoreFence() override {} | |
| static bool AddressCanHoldConstantIndex(const Object& constant, | |
| bool is_external, | |
| intptr_t cid, | |
| intptr_t index_scale); | |
| static Address ElementAddressForIntIndex(bool is_external, | |
| intptr_t cid, | |
| intptr_t index_scale, | |
| Register array, | |
| intptr_t index); | |
| static Address ElementAddressForRegIndex(bool is_external, | |
| intptr_t cid, | |
| intptr_t index_scale, | |
| bool index_unboxed, | |
| Register array, | |
| Register index); | |
| void LoadStaticFieldAddress(Register address, | |
| Register field, | |
| Register scratch, | |
| bool is_shared) { | |
| LoadCompressedSmi( | |
| scratch, compiler::FieldAddress( | |
| field, target::Field::host_offset_or_field_id_offset())); | |
| const intptr_t field_table_offset = | |
| is_shared ? compiler::target::Thread::shared_field_table_values_offset() | |
| : compiler::target::Thread::field_table_values_offset(); | |
| LoadMemoryValue(address, THR, static_cast<int32_t>(field_table_offset)); | |
| static_assert(kSmiTagShift == 1, "adjust scale factor"); | |
| leaq(address, Address(address, scratch, TIMES_HALF_WORD_SIZE, 0)); | |
| } | |
| void LoadFieldAddressForRegOffset(Register address, | |
| Register instance, | |
| Register offset_in_words_as_smi) override { | |
| static_assert(kSmiTagShift == 1, "adjust scale factor"); | |
| leaq(address, FieldAddress(instance, offset_in_words_as_smi, TIMES_4, 0)); | |
| } | |
| #if defined(DART_COMPRESSED_POINTERS) | |
| void LoadCompressedFieldAddressForRegOffset( | |
| Register address, | |
| Register instance, | |
| Register offset_in_words_as_smi) override { | |
| static_assert(kSmiTagShift == 1, "adjust scale factor"); | |
| leaq(address, FieldAddress(instance, offset_in_words_as_smi, | |
| TIMES_COMPRESSED_HALF_WORD_SIZE, 0)); | |
| } | |
| #endif | |
| void LoadFieldAddressForOffset(Register address, | |
| Register instance, | |
| int32_t offset) override { | |
| leaq(address, FieldAddress(instance, offset)); | |
| } | |
| static Address VMTagAddress(); | |
| // On some other platforms, we draw a distinction between safe and unsafe | |
| // smis. | |
| static bool IsSafe(const Object& object) { return true; } | |
| static bool IsSafeSmi(const Object& object) { return target::IsSmi(object); } | |
| void LoadWordFromPoolIndex(Register dst, intptr_t index); | |
| void StoreWordToPoolIndex(Register src, intptr_t index); | |
| private: | |
| bool constant_pool_allowed_; | |
| void CallCodeThroughPool(intptr_t target_code_pool_index, | |
| CodeEntryKind entry_kind); | |
| bool CanLoadFromObjectPool(const Object& object) const; | |
| void LoadObjectHelper( | |
| Register dst, | |
| const Object& obj, | |
| bool is_unique, | |
| ObjectPoolBuilderEntry::SnapshotBehavior snapshot_behavior = | |
| ObjectPoolBuilderEntry::kSnapshotable); | |
| void AluL(uint8_t modrm_opcode, Register dst, const Immediate& imm); | |
| void AluB(uint8_t modrm_opcode, const Address& dst, const Immediate& imm); | |
| void AluW(uint8_t modrm_opcode, const Address& dst, const Immediate& imm); | |
| void AluL(uint8_t modrm_opcode, const Address& dst, const Immediate& imm); | |
| void AluQ(uint8_t modrm_opcode, | |
| uint8_t opcode, | |
| Register dst, | |
| const Immediate& imm); | |
| void AluQ(uint8_t modrm_opcode, | |
| uint8_t opcode, | |
| const Address& dst, | |
| const Immediate& imm); | |
| void EmitSimple(int opcode, int opcode2 = -1, int opcode3 = -1); | |
| void EmitUnaryQ(Register reg, int opcode, int modrm_code); | |
| void EmitUnaryL(Register reg, int opcode, int modrm_code); | |
| void EmitUnaryQ(const Address& address, int opcode, int modrm_code); | |
| void EmitUnaryL(const Address& address, int opcode, int modrm_code); | |
| // The prefixes are in reverse order due to the rules of default arguments in | |
| // C++. | |
| void EmitQ(int reg, | |
| const Address& address, | |
| int opcode, | |
| int prefix2 = -1, | |
| int prefix1 = -1); | |
| void EmitL(int reg, | |
| const Address& address, | |
| int opcode, | |
| int prefix2 = -1, | |
| int prefix1 = -1); | |
| void EmitW(Register reg, | |
| const Address& address, | |
| int opcode, | |
| int prefix2 = -1, | |
| int prefix1 = -1); | |
| void EmitQ(int dst, int src, int opcode, int prefix2 = -1, int prefix1 = -1); | |
| void EmitL(int dst, int src, int opcode, int prefix2 = -1, int prefix1 = -1); | |
| void EmitW(Register dst, | |
| Register src, | |
| int opcode, | |
| int prefix2 = -1, | |
| int prefix1 = -1); | |
| void EmitB(int reg, const Address& address, int opcode); | |
| void CmpPS(XmmRegister dst, XmmRegister src, int condition); | |
| inline void EmitUint8(uint8_t value); | |
| inline void EmitInt32(int32_t value); | |
| inline void EmitUInt32(uint32_t value); | |
| inline void EmitInt64(int64_t value); | |
| inline void EmitRegisterREX(Register reg, | |
| uint8_t rex, | |
| bool force_emit = false); | |
| inline void EmitOperandREX(int rm, const Operand& operand, uint8_t rex); | |
| inline void EmitRegisterOperand(int rm, int reg); | |
| inline void EmitFixup(AssemblerFixup* fixup); | |
| inline void EmitOperandSizeOverride(); | |
| inline void EmitRegRegRex(int reg, int base, uint8_t rex = REX_NONE); | |
| void EmitOperand(int rm, const Operand& operand); | |
| void EmitImmediate(const Immediate& imm); | |
| void EmitComplex(int rm, const Operand& operand, const Immediate& immediate); | |
| void EmitSignExtendedInt8(int rm, | |
| const Operand& operand, | |
| const Immediate& immediate); | |
| void EmitLabel(Label* label, intptr_t instruction_size); | |
| void EmitLabelLink(Label* label); | |
| void EmitNearLabelLink(Label* label); | |
| void EmitGenericShift(bool wide, int rm, Register reg, const Immediate& imm); | |
| void EmitGenericShift(bool wide, int rm, Register operand, Register shifter); | |
| enum BarrierFilterMode { | |
| // Filter falls through into the barrier update code. Target label | |
| // is a "after-store" label. | |
| kJumpToNoUpdate, | |
| // Filter falls through to the "after-store" code. Target label | |
| // is barrier update code label. | |
| kJumpToBarrier, | |
| }; | |
| void StoreIntoArrayBarrier(Register object, | |
| Register slot, | |
| Register value, | |
| CanBeSmi can_be_smi = kValueCanBeSmi); | |
| // Unaware of write barrier (use StoreInto* methods for storing to objects). | |
| void MoveImmediate(const Address& dst, | |
| const Immediate& imm, | |
| OperandSize size = kWordBytes); | |
| friend class dart::FlowGraphCompiler; | |
| std::function<void(Register reg)> generate_invoke_write_barrier_wrapper_; | |
| std::function<void()> generate_invoke_array_write_barrier_; | |
| DISALLOW_ALLOCATION(); | |
| DISALLOW_COPY_AND_ASSIGN(Assembler); | |
| }; | |
| inline void Assembler::EmitUint8(uint8_t value) { | |
| buffer_.Emit<uint8_t>(value); | |
| } | |
| inline void Assembler::EmitInt32(int32_t value) { | |
| buffer_.Emit<int32_t>(value); | |
| } | |
| inline void Assembler::EmitUInt32(uint32_t value) { | |
| buffer_.Emit<uint32_t>(value); | |
| } | |
| inline void Assembler::EmitInt64(int64_t value) { | |
| buffer_.Emit<int64_t>(value); | |
| } | |
| inline void Assembler::EmitRegisterREX(Register reg, uint8_t rex, bool force) { | |
| ASSERT(reg != kNoRegister && reg <= R15); | |
| ASSERT(rex == REX_NONE || rex == REX_W); | |
| rex |= (reg > 7 ? REX_B : REX_NONE); | |
| if (rex != REX_NONE || force) EmitUint8(REX_PREFIX | rex); | |
| } | |
| inline void Assembler::EmitOperandREX(int rm, | |
| const Operand& operand, | |
| uint8_t rex) { | |
| rex |= (rm > 7 ? REX_R : REX_NONE) | operand.rex(); | |
| if (rex != REX_NONE) EmitUint8(REX_PREFIX | rex); | |
| } | |
| inline void Assembler::EmitRegRegRex(int reg, int base, uint8_t rex) { | |
| ASSERT(reg != kNoRegister && reg <= R15); | |
| ASSERT(base != kNoRegister && base <= R15); | |
| ASSERT(rex == REX_NONE || rex == REX_W); | |
| if (reg > 7) rex |= REX_R; | |
| if (base > 7) rex |= REX_B; | |
| if (rex != REX_NONE) EmitUint8(REX_PREFIX | rex); | |
| } | |
| inline void Assembler::EmitFixup(AssemblerFixup* fixup) { | |
| buffer_.EmitFixup(fixup); | |
| } | |
| inline void Assembler::EmitOperandSizeOverride() { | |
| EmitUint8(0x66); | |
| } | |
| } // namespace compiler | |
| } // namespace dart | |
| #endif // RUNTIME_VM_COMPILER_ASSEMBLER_ASSEMBLER_X64_H_ |