blob: b86d5c7548558c879a0b824f273e3dac4f1c1bc7 [file] [log] [blame]
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/globals.h" // NOLINT
#if defined(TARGET_ARCH_RISCV32) || defined(TARGET_ARCH_RISCV64)
#define SHOULD_NOT_INCLUDE_RUNTIME
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/cpu.h"
#include "vm/instructions.h"
#include "vm/simulator.h"
namespace dart {
DECLARE_FLAG(bool, check_code_pointer);
DECLARE_FLAG(bool, precompiled_mode);
DEFINE_FLAG(int, far_branch_level, 0, "Always use far branches");
namespace compiler {
MicroAssembler::MicroAssembler(ObjectPoolBuilder* object_pool_builder,
intptr_t far_branch_level,
ExtensionSet extensions)
: AssemblerBase(object_pool_builder),
extensions_(extensions),
far_branch_level_(far_branch_level) {
ASSERT(far_branch_level >= 0);
ASSERT(far_branch_level <= 2);
}
MicroAssembler::~MicroAssembler() {}
void MicroAssembler::Bind(Label* label) {
ASSERT(!label->IsBound());
intptr_t target_position = Position();
intptr_t branch_position;
#define BIND(head, update) \
branch_position = label->head; \
while (branch_position >= 0) { \
ASSERT(Utils::IsAligned(branch_position, Supports(RV_C) ? 2 : 4)); \
intptr_t new_offset = target_position - branch_position; \
ASSERT(Utils::IsAligned(new_offset, Supports(RV_C) ? 2 : 4)); \
intptr_t old_offset = update(branch_position, new_offset); \
if (old_offset == 0) break; \
branch_position -= old_offset; \
} \
label->head = -1
BIND(unresolved_cb_, UpdateCBOffset);
BIND(unresolved_cj_, UpdateCJOffset);
BIND(unresolved_b_, UpdateBOffset);
BIND(unresolved_j_, UpdateJOffset);
BIND(unresolved_far_, UpdateFarOffset);
label->BindTo(target_position);
}
intptr_t MicroAssembler::UpdateCBOffset(intptr_t branch_position,
intptr_t new_offset) {
CInstr instr(Read16(branch_position));
ASSERT((instr.opcode() == C_BEQZ) || (instr.opcode() == C_BNEZ));
intptr_t old_offset = instr.b_imm();
if (!IsCBImm(new_offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Write16(branch_position,
instr.opcode() | EncodeCRs1p(instr.rs1p()) | EncodeCBImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateCJOffset(intptr_t branch_position,
intptr_t new_offset) {
CInstr instr(Read16(branch_position));
ASSERT((instr.opcode() == C_J) || (instr.opcode() == C_JAL));
intptr_t old_offset = instr.j_imm();
if (!IsCJImm(new_offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Write16(branch_position, instr.opcode() | EncodeCJImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateBOffset(intptr_t branch_position,
intptr_t new_offset) {
Instr instr(Read32(branch_position));
ASSERT(instr.opcode() == BRANCH);
intptr_t old_offset = instr.btype_imm();
if (!IsBTypeImm(new_offset)) {
BailoutWithBranchOffsetError();
}
Write32(branch_position, EncodeRs2(instr.rs2()) | EncodeRs1(instr.rs1()) |
EncodeFunct3(instr.funct3()) |
EncodeOpcode(instr.opcode()) |
EncodeBTypeImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateJOffset(intptr_t branch_position,
intptr_t new_offset) {
Instr instr(Read32(branch_position));
ASSERT(instr.opcode() == JAL);
intptr_t old_offset = instr.jtype_imm();
if (!IsJTypeImm(new_offset)) {
BailoutWithBranchOffsetError();
}
Write32(branch_position, EncodeRd(instr.rd()) | EncodeOpcode(instr.opcode()) |
EncodeJTypeImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateFarOffset(intptr_t branch_position,
intptr_t new_offset) {
Instr auipc_instr(Read32(branch_position));
ASSERT(auipc_instr.opcode() == AUIPC);
ASSERT(auipc_instr.rd() == FAR_TMP);
Instr jr_instr(Read32(branch_position + 4));
ASSERT(jr_instr.opcode() == JALR);
ASSERT(jr_instr.rd() == ZR);
ASSERT(jr_instr.funct3() == F3_0);
ASSERT(jr_instr.rs1() == FAR_TMP);
intptr_t old_offset = auipc_instr.utype_imm() + jr_instr.itype_imm();
intx_t lo = new_offset << (XLEN - 12) >> (XLEN - 12);
intx_t hi = (new_offset - lo) << (XLEN - 32) >> (XLEN - 32);
if (!IsUTypeImm(hi)) {
FATAL("Jump/branch distance exceeds 2GB!");
}
Write32(branch_position,
EncodeUTypeImm(hi) | EncodeRd(FAR_TMP) | EncodeOpcode(AUIPC));
Write32(branch_position + 4, EncodeITypeImm(lo) | EncodeRs1(FAR_TMP) |
EncodeFunct3(F3_0) | EncodeRd(ZR) |
EncodeOpcode(JALR));
return old_offset;
}
void MicroAssembler::lui(Register rd, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) && (rd != ZR) && (rd != SP) && IsCUImm(imm)) {
c_lui(rd, imm);
return;
}
EmitUType(imm, rd, LUI);
}
void MicroAssembler::lui_fixed(Register rd, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitUType(imm, rd, LUI);
}
void MicroAssembler::auipc(Register rd, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitUType(imm, rd, AUIPC);
}
void MicroAssembler::jal(Register rd, Label* label, JumpDistance distance) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) &&
((distance == kNearJump) ||
(label->IsBound() && IsCJImm(label->Position() - Position())))) {
if (rd == ZR) {
c_j(label);
return;
}
#if XLEN == 32
if (rd == RA) {
c_jal(label);
return;
}
#endif // XLEN == 32
}
EmitJump(rd, label, JAL, distance);
}
void MicroAssembler::jalr(Register rd, Register rs1, intptr_t offset) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if (rs1 != ZR && offset == 0) {
if (rd == ZR) {
c_jr(rs1);
return;
} else if (rd == RA) {
c_jalr(rs1);
return;
}
}
}
EmitIType(offset, rs1, F3_0, rd, JALR);
}
void MicroAssembler::jalr_fixed(Register rd, Register rs1, intptr_t offset) {
ASSERT(Supports(RV_I));
EmitIType(offset, rs1, F3_0, rd, JALR);
}
void MicroAssembler::beq(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) &&
((distance == kNearJump) ||
(label->IsBound() && IsCBImm(label->Position() - Position())))) {
if ((rs1 == ZR) && IsCRs1p(rs2)) {
c_beqz(rs2, label);
return;
} else if ((rs2 == ZR) && IsCRs1p(rs1)) {
c_beqz(rs1, label);
return;
}
}
EmitBranch(rs1, rs2, label, BEQ, distance);
}
void MicroAssembler::bne(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) &&
((distance == kNearJump) ||
(label->IsBound() && IsCBImm(label->Position() - Position())))) {
if ((rs1 == ZR) && IsCRs1p(rs2)) {
c_bnez(rs2, label);
return;
} else if ((rs2 == ZR) && IsCRs1p(rs1)) {
c_bnez(rs1, label);
return;
}
}
EmitBranch(rs1, rs2, label, BNE, distance);
}
void MicroAssembler::blt(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
EmitBranch(rs1, rs2, label, BLT, distance);
}
void MicroAssembler::bge(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
EmitBranch(rs1, rs2, label, BGE, distance);
}
void MicroAssembler::bltu(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
EmitBranch(rs1, rs2, label, BLTU, distance);
}
void MicroAssembler::bgeu(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
EmitBranch(rs1, rs2, label, BGEU, distance);
}
void MicroAssembler::lb(Register rd, Address addr) {
ASSERT(Supports(RV_I));
EmitIType(addr.offset(), addr.base(), LB, rd, LOAD);
}
void MicroAssembler::lh(Register rd, Address addr) {
ASSERT(Supports(RV_I));
EmitIType(addr.offset(), addr.base(), LH, rd, LOAD);
}
void MicroAssembler::lw(Register rd, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (addr.base() == SP) && IsCSPLoad4Imm(addr.offset())) {
c_lwsp(rd, addr);
return;
}
if (IsCRdp(rd) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_lw(rd, addr);
return;
}
}
EmitIType(addr.offset(), addr.base(), LW, rd, LOAD);
}
void MicroAssembler::lbu(Register rd, Address addr) {
ASSERT(Supports(RV_I));
EmitIType(addr.offset(), addr.base(), LBU, rd, LOAD);
}
void MicroAssembler::lhu(Register rd, Address addr) {
ASSERT(Supports(RV_I));
EmitIType(addr.offset(), addr.base(), LHU, rd, LOAD);
}
void MicroAssembler::sb(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
EmitSType(addr.offset(), rs2, addr.base(), SB, STORE);
}
void MicroAssembler::sh(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
EmitSType(addr.offset(), rs2, addr.base(), SH, STORE);
}
void MicroAssembler::sw(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore4Imm(addr.offset())) {
c_swsp(rs2, addr);
return;
}
if (IsCRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_sw(rs2, addr);
return;
}
}
EmitSType(addr.offset(), rs2, addr.base(), SW, STORE);
}
void MicroAssembler::addi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (rs1 == ZR) && IsCIImm(imm)) {
c_li(rd, imm);
return;
}
if ((rd == rs1) && IsCIImm(imm) && (imm != 0)) {
c_addi(rd, rs1, imm);
return;
}
if ((rd == SP) && (rs1 == SP) && IsCI16Imm(imm) && (imm != 0)) {
c_addi16sp(rd, rs1, imm);
return;
}
if (IsCRdp(rd) && (rs1 == SP) && IsCI4SPNImm(imm) && (imm != 0)) {
c_addi4spn(rd, rs1, imm);
return;
}
if (imm == 0) {
if ((rd == ZR) && (rs1 == ZR)) {
c_nop();
return;
}
if ((rd != ZR) && (rs1 != ZR)) {
c_mv(rd, rs1);
return;
}
}
}
EmitIType(imm, rs1, ADDI, rd, OPIMM);
}
void MicroAssembler::slti(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitIType(imm, rs1, SLTI, rd, OPIMM);
}
void MicroAssembler::sltiu(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitIType(imm, rs1, SLTIU, rd, OPIMM);
}
void MicroAssembler::xori(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitIType(imm, rs1, XORI, rd, OPIMM);
}
void MicroAssembler::ori(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitIType(imm, rs1, ORI, rd, OPIMM);
}
void MicroAssembler::andi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCIImm(imm)) {
c_andi(rd, rs1, imm);
return;
}
}
EmitIType(imm, rs1, ANDI, rd, OPIMM);
}
void MicroAssembler::slli(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && (shamt != 0) && IsCIImm(shamt)) {
c_slli(rd, rs1, shamt);
return;
}
}
EmitRType(F7_0, shamt, rs1, SLLI, rd, OPIMM);
}
void MicroAssembler::srli(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && (shamt != 0) && IsCIImm(shamt)) {
c_srli(rd, rs1, shamt);
return;
}
}
EmitRType(F7_0, shamt, rs1, SRI, rd, OPIMM);
}
void MicroAssembler::srai(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && (shamt != 0) && IsCIImm(shamt)) {
c_srai(rd, rs1, shamt);
return;
}
}
EmitRType(SRA, shamt, rs1, SRI, rd, OPIMM);
}
void MicroAssembler::add(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if (rd == rs1) {
c_add(rd, rs1, rs2);
return;
}
if (rd == rs2) {
c_add(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, ADD, rd, OP);
}
void MicroAssembler::sub(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_sub(rd, rs1, rs2);
return;
}
}
EmitRType(SUB, rs2, rs1, ADD, rd, OP);
}
void MicroAssembler::sll(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLL, rd, OP);
}
void MicroAssembler::slt(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLT, rd, OP);
}
void MicroAssembler::sltu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLTU, rd, OP);
}
void MicroAssembler::xor_(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_xor(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_xor(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, XOR, rd, OP);
}
void MicroAssembler::srl(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SR, rd, OP);
}
void MicroAssembler::sra(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(SRA, rs2, rs1, SR, rd, OP);
}
void MicroAssembler::or_(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_or(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_or(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, OR, rd, OP);
}
void MicroAssembler::and_(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_and(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_and(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, AND, rd, OP);
}
void MicroAssembler::fence(HartEffects predecessor, HartEffects successor) {
ASSERT((predecessor & kAll) == predecessor);
ASSERT((successor & kAll) == successor);
ASSERT(Supports(RV_I));
EmitIType((predecessor << 4) | successor, ZR, FENCE, ZR, MISCMEM);
}
void MicroAssembler::fencei() {
ASSERT(Supports(RV_I));
EmitIType(0, ZR, FENCEI, ZR, MISCMEM);
}
void MicroAssembler::ecall() {
ASSERT(Supports(RV_I));
EmitIType(ECALL, ZR, F3_0, ZR, SYSTEM);
}
void MicroAssembler::ebreak() {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
c_ebreak();
return;
}
EmitIType(EBREAK, ZR, F3_0, ZR, SYSTEM);
}
void MicroAssembler::SimulatorPrintObject(Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(ECALL, rs1, F3_0, ZR, SYSTEM);
}
void MicroAssembler::csrrw(Register rd, uint32_t csr, Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(csr, rs1, CSRRW, rd, SYSTEM);
}
void MicroAssembler::csrrs(Register rd, uint32_t csr, Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(csr, rs1, CSRRS, rd, SYSTEM);
}
void MicroAssembler::csrrc(Register rd, uint32_t csr, Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(csr, rs1, CSRRC, rd, SYSTEM);
}
void MicroAssembler::csrrwi(Register rd, uint32_t csr, uint32_t imm) {
ASSERT(Supports(RV_I));
EmitIType(csr, Register(imm), CSRRWI, rd, SYSTEM);
}
void MicroAssembler::csrrsi(Register rd, uint32_t csr, uint32_t imm) {
ASSERT(Supports(RV_I));
EmitIType(csr, Register(imm), CSRRSI, rd, SYSTEM);
}
void MicroAssembler::csrrci(Register rd, uint32_t csr, uint32_t imm) {
ASSERT(Supports(RV_I));
EmitIType(csr, Register(imm), CSRRCI, rd, SYSTEM);
}
void MicroAssembler::trap() {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
Emit16(0); // Permanently reserved illegal instruction.
} else {
Emit32(0); // Permanently reserved illegal instruction.
}
}
#if XLEN >= 64
void MicroAssembler::lwu(Register rd, Address addr) {
ASSERT(Supports(RV_I));
EmitIType(addr.offset(), addr.base(), LWU, rd, LOAD);
}
void MicroAssembler::ld(Register rd, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (addr.base() == SP) && IsCSPLoad8Imm(addr.offset())) {
c_ldsp(rd, addr);
return;
}
if (IsCRdp(rd) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_ld(rd, addr);
return;
}
}
EmitIType(addr.offset(), addr.base(), LD, rd, LOAD);
}
void MicroAssembler::sd(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore8Imm(addr.offset())) {
c_sdsp(rs2, addr);
return;
}
if (IsCRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_sd(rs2, addr);
return;
}
}
EmitSType(addr.offset(), rs2, addr.base(), SD, STORE);
}
void MicroAssembler::addiw(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (rs1 == ZR) && IsCIImm(imm)) {
c_li(rd, imm);
return;
}
if ((rd == rs1) && (rd != ZR) && IsCIImm(imm)) {
c_addiw(rd, rs1, imm);
return;
}
}
EmitIType(imm, rs1, ADDI, rd, OPIMM32);
}
void MicroAssembler::slliw(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < 32));
ASSERT(Supports(RV_I));
EmitRType(F7_0, shamt, rs1, SLLI, rd, OPIMM32);
}
void MicroAssembler::srliw(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < 32));
ASSERT(Supports(RV_I));
EmitRType(F7_0, shamt, rs1, SRI, rd, OPIMM32);
}
void MicroAssembler::sraiw(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
EmitRType(SRA, shamt, rs1, SRI, rd, OPIMM32);
}
void MicroAssembler::addw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_addw(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_addw(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, ADD, rd, OP32);
}
void MicroAssembler::subw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_subw(rd, rs1, rs2);
return;
}
}
EmitRType(SUB, rs2, rs1, ADD, rd, OP32);
}
void MicroAssembler::sllw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLL, rd, OP32);
}
void MicroAssembler::srlw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SR, rd, OP32);
}
void MicroAssembler::sraw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(SRA, rs2, rs1, SR, rd, OP32);
}
#endif // XLEN >= 64
void MicroAssembler::mul(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MUL, rd, OP);
}
void MicroAssembler::mulh(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULH, rd, OP);
}
void MicroAssembler::mulhsu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULHSU, rd, OP);
}
void MicroAssembler::mulhu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULHU, rd, OP);
}
void MicroAssembler::div(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIV, rd, OP);
}
void MicroAssembler::divu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIVU, rd, OP);
}
void MicroAssembler::rem(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REM, rd, OP);
}
void MicroAssembler::remu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REMU, rd, OP);
}
#if XLEN >= 64
void MicroAssembler::mulw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULW, rd, OP32);
}
void MicroAssembler::divw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIVW, rd, OP32);
}
void MicroAssembler::divuw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIVUW, rd, OP32);
}
void MicroAssembler::remw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REMW, rd, OP32);
}
void MicroAssembler::remuw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REMUW, rd, OP32);
}
#endif // XLEN >= 64
void MicroAssembler::lrw(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(LR, order, ZR, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::scw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(SC, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoswapw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOSWAP, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoaddw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOADD, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoxorw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOXOR, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoandw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOAND, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoorw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOOR, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amominw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMIN, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amomaxw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAX, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amominuw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMINU, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amomaxuw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAXU, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
#if XLEN >= 64
void MicroAssembler::lrd(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(LR, order, ZR, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::scd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(SC, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoswapd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOSWAP, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoaddd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOADD, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoxord(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOXOR, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoandd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOAND, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoord(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOOR, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amomind(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMIN, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amomaxd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAX, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amominud(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMINU, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amomaxud(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAXU, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
#endif // XLEN >= 64
void MicroAssembler::flw(FRegister rd, Address addr) {
ASSERT(Supports(RV_F));
#if XLEN == 32
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPLoad4Imm(addr.offset())) {
c_flwsp(rd, addr);
return;
}
if (IsCFRdp(rd) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_flw(rd, addr);
return;
}
}
#endif // XLEN == 32
EmitIType(addr.offset(), addr.base(), S, rd, LOADFP);
}
void MicroAssembler::fsw(FRegister rs2, Address addr) {
ASSERT(Supports(RV_F));
#if XLEN == 32
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore4Imm(addr.offset())) {
c_fswsp(rs2, addr);
return;
}
if (IsCFRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_fsw(rs2, addr);
return;
}
}
#endif // XLEN == 32
EmitSType(addr.offset(), rs2, addr.base(), S, STOREFP);
}
void MicroAssembler::fmadds(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FMADD);
}
void MicroAssembler::fmsubs(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FMSUB);
}
void MicroAssembler::fnmsubs(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FNMSUB);
}
void MicroAssembler::fnmadds(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FNMADD);
}
void MicroAssembler::fadds(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FADDS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsubs(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FSUBS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmuls(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FMULS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fdivs(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FDIVS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsqrts(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FSQRTS, FRegister(0), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsgnjs(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FSGNJS, rs2, rs1, J, rd, OPFP);
}
void MicroAssembler::fsgnjns(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FSGNJS, rs2, rs1, JN, rd, OPFP);
}
void MicroAssembler::fsgnjxs(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FSGNJS, rs2, rs1, JX, rd, OPFP);
}
void MicroAssembler::fmins(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FMINMAXS, rs2, rs1, MIN, rd, OPFP);
}
void MicroAssembler::fmaxs(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FMINMAXS, rs2, rs1, MAX, rd, OPFP);
}
void MicroAssembler::feqs(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FCMPS, rs2, rs1, FEQ, rd, OPFP);
}
void MicroAssembler::flts(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FCMPS, rs2, rs1, FLT, rd, OPFP);
}
void MicroAssembler::fles(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FCMPS, rs2, rs1, FLE, rd, OPFP);
}
void MicroAssembler::fclasss(Register rd, FRegister rs1) {
ASSERT(Supports(RV_F));
EmitRType(FCLASSS, FRegister(0), rs1, F3_1, rd, OPFP);
}
void MicroAssembler::fcvtws(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtwus(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(WU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtsw(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, FRegister(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtswu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, FRegister(WU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmvxw(Register rd, FRegister rs1) {
ASSERT(Supports(RV_F));
EmitRType(FMVXW, FRegister(0), rs1, F3_0, rd, OPFP);
}
void MicroAssembler::fmvwx(FRegister rd, Register rs1) {
ASSERT(Supports(RV_F));
EmitRType(FMVWX, FRegister(0), rs1, F3_0, rd, OPFP);
}
#if XLEN >= 64
void MicroAssembler::fcvtls(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtlus(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(LU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtsl(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, FRegister(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtslu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, FRegister(LU), rs1, rounding, rd, OPFP);
}
#endif // XLEN >= 64
void MicroAssembler::fld(FRegister rd, Address addr) {
ASSERT(Supports(RV_D));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPLoad8Imm(addr.offset())) {
c_fldsp(rd, addr);
return;
}
if (IsCFRdp(rd) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_fld(rd, addr);
return;
}
}
EmitIType(addr.offset(), addr.base(), D, rd, LOADFP);
}
void MicroAssembler::fsd(FRegister rs2, Address addr) {
ASSERT(Supports(RV_D));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore8Imm(addr.offset())) {
c_fsdsp(rs2, addr);
return;
}
if (IsCFRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_fsd(rs2, addr);
return;
}
}
EmitSType(addr.offset(), rs2, addr.base(), D, STOREFP);
}
void MicroAssembler::fmaddd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FMADD);
}
void MicroAssembler::fmsubd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FMSUB);
}
void MicroAssembler::fnmsubd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FNMSUB);
}
void MicroAssembler::fnmaddd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FNMADD);
}
void MicroAssembler::faddd(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FADDD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsubd(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FSUBD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmuld(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FMULD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fdivd(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FDIVD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsqrtd(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FSQRTD, FRegister(0), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsgnjd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FSGNJD, rs2, rs1, J, rd, OPFP);
}
void MicroAssembler::fsgnjnd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FSGNJD, rs2, rs1, JN, rd, OPFP);
}
void MicroAssembler::fsgnjxd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FSGNJD, rs2, rs1, JX, rd, OPFP);
}
void MicroAssembler::fmind(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FMINMAXD, rs2, rs1, MIN, rd, OPFP);
}
void MicroAssembler::fmaxd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FMINMAXD, rs2, rs1, MAX, rd, OPFP);
}
void MicroAssembler::fcvtsd(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTS, FRegister(1), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtds(FRegister rd, FRegister rs1) {
ASSERT(Supports(RV_D));
EmitRType(FCVTD, FRegister(0), rs1, F3_0, rd, OPFP);
}
void MicroAssembler::feqd(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FCMPD, rs2, rs1, FEQ, rd, OPFP);
}
void MicroAssembler::fltd(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FCMPD, rs2, rs1, FLT, rd, OPFP);
}
void MicroAssembler::fled(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FCMPD, rs2, rs1, FLE, rd, OPFP);
}
void MicroAssembler::fclassd(Register rd, FRegister rs1) {
ASSERT(Supports(RV_D));
EmitRType(FCLASSD, FRegister(0), rs1, F3_1, rd, OPFP);
}
void MicroAssembler::fcvtwd(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtwud(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(WU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtdw(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, FRegister(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtdwu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, FRegister(WU), rs1, rounding, rd, OPFP);
}
#if XLEN >= 64
void MicroAssembler::fcvtld(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtlud(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(LU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmvxd(Register rd, FRegister rs1) {
ASSERT(Supports(RV_D));
EmitRType(FMVXD, FRegister(0), rs1, F3_0, rd, OPFP);
}
void MicroAssembler::fcvtdl(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, FRegister(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtdlu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, FRegister(LU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmvdx(FRegister rd, Register rs1) {
ASSERT(Supports(RV_D));
EmitRType(FMVDX, FRegister(0), rs1, F3_0, rd, OPFP);
}
#endif // XLEN >= 64
void MicroAssembler::c_lwsp(Register rd, Address addr) {
ASSERT(rd != ZR);
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_LWSP | EncodeCRd(rd) | EncodeCSPLoad4Imm(addr.offset()));
}
#if XLEN == 32
void MicroAssembler::c_flwsp(FRegister rd, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FLWSP | EncodeCFRd(rd) | EncodeCSPLoad4Imm(addr.offset()));
}
#else
void MicroAssembler::c_ldsp(Register rd, Address addr) {
ASSERT(rd != ZR);
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_LDSP | EncodeCRd(rd) | EncodeCSPLoad8Imm(addr.offset()));
}
#endif
void MicroAssembler::c_fldsp(FRegister rd, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FLDSP | EncodeCFRd(rd) | EncodeCSPLoad8Imm(addr.offset()));
}
void MicroAssembler::c_swsp(Register rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_SWSP | EncodeCRs2(rs2) | EncodeCSPStore4Imm(addr.offset()));
}
#if XLEN == 32
void MicroAssembler::c_fswsp(FRegister rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FSWSP | EncodeCFRs2(rs2) | EncodeCSPStore4Imm(addr.offset()));
}
#else
void MicroAssembler::c_sdsp(Register rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_SDSP | EncodeCRs2(rs2) | EncodeCSPStore8Imm(addr.offset()));
}
#endif
void MicroAssembler::c_fsdsp(FRegister rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FSDSP | EncodeCFRs2(rs2) | EncodeCSPStore8Imm(addr.offset()));
}
void MicroAssembler::c_lw(Register rd, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_LW | EncodeCRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_ld(Register rd, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_LD | EncodeCRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_flw(FRegister rd, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FLW | EncodeCFRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_fld(FRegister rd, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FLD | EncodeCFRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_sw(Register rs2, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_SW | EncodeCRs1p(addr.base()) | EncodeCRs2p(rs2) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_sd(Register rs2, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_SD | EncodeCRs1p(addr.base()) | EncodeCRs2p(rs2) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_fsw(FRegister rs2, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FSW | EncodeCRs1p(addr.base()) | EncodeCFRs2p(rs2) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_fsd(FRegister rs2, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FSD | EncodeCRs1p(addr.base()) | EncodeCFRs2p(rs2) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_j(Label* label) {
ASSERT(Supports(RV_C));
EmitCJump(label, C_J);
}
#if XLEN == 32
void MicroAssembler::c_jal(Label* label) {
ASSERT(Supports(RV_C));
EmitCJump(label, C_JAL);
}
#endif // XLEN == 32
void MicroAssembler::c_jr(Register rs1) {
ASSERT(Supports(RV_C));
ASSERT(rs1 != ZR);
Emit16(C_JR | EncodeCRs1(rs1) | EncodeCRs2(ZR));
}
void MicroAssembler::c_jalr(Register rs1) {
ASSERT(Supports(RV_C));
Emit16(C_JALR | EncodeCRs1(rs1) | EncodeCRs2(ZR));
}
void MicroAssembler::c_beqz(Register rs1p, Label* label) {
ASSERT(Supports(RV_C));
EmitCBranch(rs1p, label, C_BEQZ);
}
void MicroAssembler::c_bnez(Register rs1p, Label* label) {
ASSERT(Supports(RV_C));
EmitCBranch(rs1p, label, C_BNEZ);
}
void MicroAssembler::c_li(Register rd, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
Emit16(C_LI | EncodeCRd(rd) | EncodeCIImm(imm));
}
void MicroAssembler::c_lui(Register rd, uintptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
ASSERT(rd != SP);
Emit16(C_LUI | EncodeCRd(rd) | EncodeCUImm(imm));
}
void MicroAssembler::c_addi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(imm != 0);
ASSERT(rd == rs1);
Emit16(C_ADDI | EncodeCRd(rd) | EncodeCIImm(imm));
}
#if XLEN >= 64
void MicroAssembler::c_addiw(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_ADDIW | EncodeCRd(rd) | EncodeCIImm(imm));
}
#endif
void MicroAssembler::c_addi16sp(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_ADDI16SP | EncodeCRd(rd) | EncodeCI16Imm(imm));
}
void MicroAssembler::c_addi4spn(Register rdp, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rs1 == SP);
ASSERT(imm != 0);
Emit16(C_ADDI4SPN | EncodeCRdp(rdp) | EncodeCI4SPNImm(imm));
}
void MicroAssembler::c_slli(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
ASSERT(imm != 0);
Emit16(C_SLLI | EncodeCRd(rd) | EncodeCIImm(imm));
}
void MicroAssembler::c_srli(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
ASSERT(imm != 0);
Emit16(C_SRLI | EncodeCRs1p(rd) | EncodeCIImm(imm));
}
void MicroAssembler::c_srai(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
ASSERT(imm != 0);
Emit16(C_SRAI | EncodeCRs1p(rd) | EncodeCIImm(imm));
}
void MicroAssembler::c_andi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_ANDI | EncodeCRs1p(rd) | EncodeCIImm(imm));
}
void MicroAssembler::c_mv(Register rd, Register rs2) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
ASSERT(rs2 != ZR);
Emit16(C_MV | EncodeCRd(rd) | EncodeCRs2(rs2));
}
void MicroAssembler::c_add(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
ASSERT(rd == rs1);
ASSERT(rs2 != ZR);
Emit16(C_ADD | EncodeCRd(rd) | EncodeCRs2(rs2));
}
void MicroAssembler::c_and(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_AND | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_or(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_OR | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_xor(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_XOR | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_sub(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_SUB | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
#if XLEN >= 64
void MicroAssembler::c_addw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_ADDW | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_subw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_SUBW | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
#endif // XLEN >= 64
void MicroAssembler::c_nop() {
ASSERT(Supports(RV_C));
Emit16(C_NOP);
}
void MicroAssembler::c_ebreak() {
ASSERT(Supports(RV_C));
Emit16(C_EBREAK);
}
static Funct3 InvertFunct3(Funct3 func) {
switch (func) {
case BEQ:
return BNE;
case BNE:
return BEQ;
case BGE:
return BLT;
case BGEU:
return BLTU;
case BLT:
return BGE;
case BLTU:
return BGEU;
default:
UNREACHABLE();
}
}
void MicroAssembler::EmitBranch(Register rs1,
Register rs2,
Label* label,
Funct3 func,
JumpDistance distance) {
intptr_t offset;
if (label->IsBound()) {
// Backward branch: use near or far branch based on actual distance.
offset = label->Position() - Position();
if (IsBTypeImm(offset)) {
EmitBType(offset, rs2, rs1, func, BRANCH);
return;
}
if (IsJTypeImm(offset + 4)) {
intptr_t start = Position();
const intptr_t kFarBranchLength = 8;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
offset = label->Position() - Position();
EmitJType(offset, ZR, JAL);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
return;
}
intptr_t start = Position();
const intptr_t kFarBranchLength = 12;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
offset = label->Position() - Position();
intx_t lo = offset << (XLEN - 12) >> (XLEN - 12);
intx_t hi = (offset - lo) << (XLEN - 32) >> (XLEN - 32);
if (!IsUTypeImm(hi)) {
FATAL("Branch distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
return;
} else {
// Forward branch: speculatively use near branches and re-assemble with far
// branches if any need greater length.
if (distance == kNearJump) {
offset = label->link_b(Position());
if (!IsBTypeImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
EmitBType(offset, rs2, rs1, func, BRANCH);
} else if (far_branch_level() == 0) {
offset = label->link_b(Position());
if (!IsBTypeImm(offset)) {
// TODO(riscv): This isn't so much because the branch is out of range
// as some previous jump to the same target would be out of B-type
// range... A possible alternative is to have separate lists on Labels
// for pending B-type and J-type instructions.
BailoutWithBranchOffsetError();
}
EmitBType(offset, rs2, rs1, func, BRANCH);
} else if (far_branch_level() == 1) {
intptr_t start = Position();
const intptr_t kFarBranchLength = 8;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
offset = label->link_j(Position());
EmitJType(offset, ZR, JAL);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
} else {
intptr_t start = Position();
const intptr_t kFarBranchLength = 12;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
offset = label->link_far(Position());
intx_t lo = offset << (XLEN - 12) >> (XLEN - 12);
intx_t hi = (offset - lo) << (XLEN - 32) >> (XLEN - 32);
if (!IsUTypeImm(hi)) {
FATAL("Branch distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
}
}
}
void MicroAssembler::EmitJump(Register rd,
Label* label,
Opcode op,
JumpDistance distance) {
intptr_t offset;
if (label->IsBound()) {
// Backward jump: use near or far jump based on actual distance.
offset = label->Position() - Position();
if (IsJTypeImm(offset)) {
EmitJType(offset, rd, JAL);
return;
}
intx_t lo = offset << (XLEN - 12) >> (XLEN - 12);
intx_t hi = (offset - lo) << (XLEN - 32) >> (XLEN - 32);
if (!IsUTypeImm(hi)) {
FATAL("Jump distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
return;
} else {
// Forward jump: speculatively use near jumps and re-assemble with far
// jumps if any need greater length.
if (distance == kNearJump) {
offset = label->link_j(Position());
if (!IsJTypeImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
EmitJType(offset, rd, JAL);
} else if (far_branch_level() < 2) {
offset = label->link_j(Position());
if (!IsJTypeImm(offset)) {
BailoutWithBranchOffsetError();
}
EmitJType(offset, rd, JAL);
} else {
offset = label->link_far(Position());
intx_t lo = offset << (XLEN - 12) >> (XLEN - 12);
intx_t hi = (offset - lo) << (XLEN - 32) >> (XLEN - 32);
if (!IsUTypeImm(hi)) {
FATAL("Jump distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
}
}
}
void MicroAssembler::EmitCBranch(Register rs1p, Label* label, COpcode op) {
intptr_t offset;
if (label->IsBound()) {
offset = label->Position() - Position();
} else {
offset = label->link_cb(Position());
}
if (!IsCBImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Emit16(op | EncodeCRs1p(rs1p) | EncodeCBImm(offset));
}
void MicroAssembler::EmitCJump(Label* label, COpcode op) {
intptr_t offset;
if (label->IsBound()) {
offset = label->Position() - Position();
} else {
offset = label->link_cj(Position());
}
if (!IsCJImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Emit16(op | EncodeCJImm(offset));
}
void MicroAssembler::EmitRType(Funct5 funct5,
std::memory_order order,
Register rs2,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
intptr_t funct7 = funct5 << 2;
switch (order) {
case std::memory_order_acq_rel:
funct7 |= 0b11;
break;
case std::memory_order_acquire:
funct7 |= 0b10;
break;
case std::memory_order_release:
funct7 |= 0b01;
break;
case std::memory_order_relaxed:
funct7 |= 0b00;
break;
default:
FATAL("Invalid memory order");
}
EmitRType((Funct7)funct7, rs2, rs1, funct3, rd, opcode);
}
void MicroAssembler::EmitRType(Funct7 funct7,
Register rs2,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
Funct3 funct3,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
RoundingMode round,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
Register rs1,
RoundingMode round,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
Register rs1,
Funct3 funct3,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
RoundingMode round,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
intptr_t shamt,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeShamt(shamt);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitR4Type(FRegister rs3,
Funct2 funct2,
FRegister rs2,
FRegister rs1,
RoundingMode round,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFRs3(rs3);
e |= EncodeFunct2(funct2);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitIType(intptr_t imm,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeITypeImm(imm);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitIType(intptr_t imm,
Register rs1,
Funct3 funct3,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeITypeImm(imm);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitSType(intptr_t imm,
Register rs2,
Register rs1,
Funct3 funct3,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeSTypeImm(imm);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitSType(intptr_t imm,
FRegister rs2,
Register rs1,
Funct3 funct3,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeSTypeImm(imm);
e |= EncodeFRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitBType(intptr_t imm,
Register rs2,
Register rs1,
Funct3 funct3,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeBTypeImm(imm);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitUType(intptr_t imm, Register rd, Opcode opcode) {
uint32_t e = 0;
e |= EncodeUTypeImm(imm);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitJType(intptr_t imm, Register rd, Opcode opcode) {
uint32_t e = 0;
e |= EncodeJTypeImm(imm);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
Assembler::Assembler(ObjectPoolBuilder* object_pool_builder,
intptr_t far_branch_level)
: MicroAssembler(object_pool_builder,
far_branch_level,
FLAG_use_compressed_instructions ? RV_GC : RV_G),
constant_pool_allowed_(false) {
generate_invoke_write_barrier_wrapper_ = [&](Register reg) {
// Note this does not destory RA.
lx(TMP,
Address(THR, target::Thread::write_barrier_wrappers_thread_offset(reg)));
jalr(TMP, TMP);
};
generate_invoke_array_write_barrier_ = [&]() {
Call(
Address(THR, target::Thread::array_write_barrier_entry_point_offset()));
};
}
void Assembler::PushRegister(Register r) {
ASSERT(r != SP);
subi(SP, SP, target::kWordSize);
sx(r, Address(SP, 0));
}
void Assembler::PopRegister(Register r) {
ASSERT(r != SP);
lx(r, Address(SP, 0));
addi(SP, SP, target::kWordSize);
}
void Assembler::PushRegisterPair(Register r0, Register r1) {
ASSERT(r0 != SP);
ASSERT(r1 != SP);
subi(SP, SP, 2 * target::kWordSize);
sx(r1, Address(SP, target::kWordSize));
sx(r0, Address(SP, 0));
}
void Assembler::PopRegisterPair(Register r0, Register r1) {
ASSERT(r0 != SP);
ASSERT(r1 != SP);
lx(r1, Address(SP, target::kWordSize));
lx(r0, Address(SP, 0));
addi(SP, SP, 2 * target::kWordSize);
}
void Assembler::PushRegisters(const RegisterSet& regs) {
// The order in which the registers are pushed must match the order
// in which the registers are encoded in the safepoint's stack map.
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
(regs.FpuRegisterCount() * kFpuRegisterSize);
if (size == 0) {
return; // Skip no-op SP update.
}
subi(SP, SP, size);
intptr_t offset = size;
for (intptr_t i = kNumberOfFpuRegisters - 1; i >= 0; i--) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
offset -= kFpuRegisterSize;
fsd(reg, Address(SP, offset));
}
}
for (intptr_t i = kNumberOfCpuRegisters - 1; i >= 0; i--) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
offset -= target::kWordSize;
sx(reg, Address(SP, offset));
}
}
ASSERT(offset == 0);
}
void Assembler::PopRegisters(const RegisterSet& regs) {
// The order in which the registers are pushed must match the order
// in which the registers are encoded in the safepoint's stack map.
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
(regs.FpuRegisterCount() * kFpuRegisterSize);
if (size == 0) {
return; // Skip no-op SP update.
}
intptr_t offset = 0;
for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
lx(reg, Address(SP, offset));
offset += target::kWordSize;
}
}
for (intptr_t i = 0; i < kNumberOfFpuRegisters; i++) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
fld(reg, Address(SP, offset));
offset += kFpuRegisterSize;
}
}
ASSERT(offset == size);
addi(SP, SP, size);
}
void Assembler::PushNativeCalleeSavedRegisters() {
RegisterSet regs(kAbiPreservedCpuRegs, kAbiPreservedFpuRegs);
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
(regs.FpuRegisterCount() * sizeof(double));
subi(SP, SP, size);
intptr_t offset = 0;
for (intptr_t i = 0; i < kNumberOfFpuRegisters; i++) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
fsd(reg, Address(SP, offset));
offset += sizeof(double);
}
}
for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
sx(reg, Address(SP, offset));
offset += target::kWordSize;
}
}
ASSERT(offset == size);
}
void Assembler::PopNativeCalleeSavedRegisters() {
RegisterSet regs(kAbiPreservedCpuRegs, kAbiPreservedFpuRegs);
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
(regs.FpuRegisterCount() * sizeof(double));
intptr_t offset = 0;
for (intptr_t i = 0; i < kNumberOfFpuRegisters; i++) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
fld(reg, Address(SP, offset));
offset += sizeof(double);
}
}
for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
lx(reg, Address(SP, offset));
offset += target::kWordSize;
}
}
ASSERT(offset == size);
addi(SP, SP, size);
}
void Assembler::ExtendValue(Register rd, Register rn, OperandSize sz) {
switch (sz) {
#if XLEN == 64
case kEightBytes:
if (rd == rn) return; // No operation needed.
return mv(rd, rn);
case kUnsignedFourBytes:
return UNIMPLEMENTED();
case kFourBytes:
return sextw(rd, rn);
#elif XLEN == 32
case kUnsignedFourBytes:
case kFourBytes:
if (rd == rn) return; // No operation needed.
return mv(rd, rn);
#endif
case kUnsignedTwoBytes:
case kTwoBytes:
case kUnsignedByte:
case kByte:
default:
UNIMPLEMENTED();
break;
}
UNIMPLEMENTED();
}
void Assembler::ExtendAndSmiTagValue(Register rd, Register rn, OperandSize sz) {
if (sz == kWordBytes) {
SmiTag(rd, rn);
return;
}
switch (sz) {
#if XLEN == 64
case kUnsignedFourBytes:
slli(rd, rn, XLEN - kBitsPerInt32);
srli(rd, rd, XLEN - kBitsPerInt32 - kSmiTagShift);
return;
case kFourBytes:
slli(rd, rn, XLEN - kBitsPerInt32);
srai(rd, rd, XLEN - kBitsPerInt32 - kSmiTagShift);
return;
#endif
case kUnsignedTwoBytes:
slli(rd, rn, XLEN - kBitsPerInt16);
srli(rd, rd, XLEN - kBitsPerInt16 - kSmiTagShift);
return;
case kTwoBytes:
slli(rd, rn, XLEN - kBitsPerInt16);
srai(rd, rd, XLEN - kBitsPerInt16 - kSmiTagShift);
return;
case kUnsignedByte:
slli(rd, rn, XLEN - kBitsPerInt8);
srli(rd, rd, XLEN - kBitsPerInt8 - kSmiTagShift);
return;
case kByte:
slli(rd, rn, XLEN - kBitsPerInt8);
srai(rd, rd, XLEN - kBitsPerInt8 - kSmiTagShift);
return;
default:
UNIMPLEMENTED();
break;
}
}
// Unconditional jump to a given address in memory. Clobbers TMP.
void Assembler::Jump(const Address& address) {
lx(TMP2, address);
jr(TMP2);
}
void Assembler::LoadField(Register dst, const FieldAddress& address) {
lx(dst, address);
}
#if defined(USING_THREAD_SANITIZER)
void Assembler::TsanLoadAcquire(Register addr) {
UNIMPLEMENTED();
}
void Assembler::TsanStoreRelease(Register addr) {
UNIMPLEMENTED();
}
#endif
void Assembler::LoadAcquire(Register dst, Register address, int32_t offset) {
ASSERT(dst != address);
LoadFromOffset(dst, address, offset);
fence(HartEffects::kRead, HartEffects::kMemory);
#if defined(USING_THREAD_SANITIZER)
if (offset == 0) {
TsanLoadAcquire(address);
} else {
AddImmediate(TMP2, address, offset);
TsanLoadAcquire(TMP2);
}
#endif
}
void Assembler::LoadAcquireCompressed(Register dst,
Register address,
int32_t offset) {
LoadAcquire(dst, address, offset);
}
void Assembler::StoreRelease(Register src, Register address, int32_t offset) {
fence(HartEffects::kMemory, HartEffects::kRead);
StoreToOffset(src, address, offset);
}
void Assembler::StoreReleaseCompressed(Register src,
Register address,
int32_t offset) {
UNIMPLEMENTED();
}
void Assembler::CompareWithCompressedFieldFromOffset(Register value,
Register base,
int32_t offset) {
UNIMPLEMENTED();
}
void Assembler::CompareWithMemoryValue(Register value,
Address address,
OperandSize sz) {
UNIMPLEMENTED();
}
void Assembler::CompareFunctionTypeNullabilityWith(Register type,
int8_t value) {
EnsureHasClassIdInDEBUG(kFunctionTypeCid, type, TMP);
lbu(TMP,
FieldAddress(type, compiler::target::FunctionType::nullability_offset()));
CompareImmediate(TMP, value);
}
void Assembler::CompareTypeNullabilityWith(Register type, int8_t value) {
EnsureHasClassIdInDEBUG(kTypeCid, type, TMP);
lbu(TMP, FieldAddress(type, compiler::target::Type::nullability_offset()));
CompareImmediate(TMP, value);
}
void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
if (frame_space != 0) {
addi(SP, SP, -frame_space);
}
const intptr_t kAbiStackAlignment = 16; // For both 32 and 64 bit.
andi(SP, SP, ~(kAbiStackAlignment - 1));
}
// In debug mode, this generates code to check that:
// FP + kExitLinkSlotFromEntryFp == SP
// or triggers breakpoint otherwise.
void Assembler::EmitEntryFrameVerification() {
#if defined(DEBUG)
Label done;
ASSERT(!constant_pool_allowed());
LoadImmediate(TMP, target::frame_layout.exit_link_slot_from_entry_fp *
target::kWordSize);
add(TMP, TMP, FPREG);
beq(TMP, SPREG, &done, kNearJump);
Breakpoint();
Bind(&done);
#endif
}
void Assembler::CompareRegisters(Register rn, Register rm) {
ASSERT(deferred_compare_ == kNone);
deferred_compare_ = kCompareReg;
deferred_left_ = rn;
deferred_reg_ = rm;
}
void Assembler::CompareObjectRegisters(Register rn, Register rm) {