Google Git
Sign in
dart / sdk.git / refs/heads/main / . / runtime / vm / compiler / assembler / assembler_riscv.cc
blob: df6abd916aca92940269635e366fcd6235987fdf [file] [log] [blame] [edit]
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/globals.h" // NOLINT
#if defined(TARGET_ARCH_RISCV32) || defined(TARGET_ARCH_RISCV64)
#define SHOULD_NOT_INCLUDE_RUNTIME
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/cpu.h"
#include "vm/instructions.h"
#include "vm/simulator.h"
#include "vm/tags.h"
namespace dart {
DECLARE_FLAG(bool, check_code_pointer);
DECLARE_FLAG(bool, precompiled_mode);
DEFINE_FLAG(int, far_branch_level, 0, "Always use far branches");
namespace compiler {
MicroAssembler::MicroAssembler(ObjectPoolBuilder* object_pool_builder,
intptr_t far_branch_level,
ExtensionSet extensions)
: AssemblerBase(object_pool_builder),
extensions_(extensions),
far_branch_level_(far_branch_level) {
ASSERT(far_branch_level >= 0);
ASSERT(far_branch_level <= 2);
}
MicroAssembler::~MicroAssembler() {}
void MicroAssembler::Bind(Label* label) {
ASSERT(!label->IsBound());
intptr_t target_position = Position();
intptr_t branch_position;
#define BIND(head, update) \
branch_position = label->head; \
while (branch_position >= 0) { \
ASSERT(Utils::IsAligned(branch_position, Supports(RV_C) ? 2 : 4)); \
intptr_t new_offset = target_position - branch_position; \
ASSERT(Utils::IsAligned(new_offset, Supports(RV_C) ? 2 : 4)); \
intptr_t old_offset = update(branch_position, new_offset); \
if (old_offset == 0) break; \
branch_position -= old_offset; \
} \
label->head = -1
BIND(unresolved_cb_, UpdateCBOffset);
BIND(unresolved_cj_, UpdateCJOffset);
BIND(unresolved_b_, UpdateBOffset);
BIND(unresolved_j_, UpdateJOffset);
BIND(unresolved_far_, UpdateFarOffset);
label->BindTo(target_position);
}
intptr_t MicroAssembler::UpdateCBOffset(intptr_t branch_position,
intptr_t new_offset) {
CInstr instr(Read16(branch_position));
ASSERT((instr.opcode() == C_BEQZ) || (instr.opcode() == C_BNEZ));
intptr_t old_offset = instr.b_imm();
if (!IsCBImm(new_offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Write16(branch_position,
instr.opcode() | EncodeCRs1p(instr.rs1p()) | EncodeCBImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateCJOffset(intptr_t branch_position,
intptr_t new_offset) {
CInstr instr(Read16(branch_position));
ASSERT((instr.opcode() == C_J) || (instr.opcode() == C_JAL));
intptr_t old_offset = instr.j_imm();
if (!IsCJImm(new_offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Write16(branch_position, instr.opcode() | EncodeCJImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateBOffset(intptr_t branch_position,
intptr_t new_offset) {
Instr instr(Read32(branch_position));
ASSERT(instr.opcode() == BRANCH);
intptr_t old_offset = instr.btype_imm();
if (!IsBTypeImm(new_offset)) {
BailoutWithBranchOffsetError();
}
Write32(branch_position, EncodeRs2(instr.rs2()) | EncodeRs1(instr.rs1()) |
EncodeFunct3(instr.funct3()) |
EncodeOpcode(instr.opcode()) |
EncodeBTypeImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateJOffset(intptr_t branch_position,
intptr_t new_offset) {
Instr instr(Read32(branch_position));
ASSERT(instr.opcode() == JAL);
intptr_t old_offset = instr.jtype_imm();
if (!IsJTypeImm(new_offset)) {
BailoutWithBranchOffsetError();
}
Write32(branch_position, EncodeRd(instr.rd()) | EncodeOpcode(instr.opcode()) |
EncodeJTypeImm(new_offset));
return old_offset;
}
intptr_t MicroAssembler::UpdateFarOffset(intptr_t branch_position,
intptr_t new_offset) {
Instr auipc_instr(Read32(branch_position));
ASSERT(auipc_instr.opcode() == AUIPC);
ASSERT(auipc_instr.rd() == FAR_TMP);
Instr jr_instr(Read32(branch_position + 4));
ASSERT(jr_instr.opcode() == JALR);
ASSERT(jr_instr.rd() == ZR);
ASSERT(jr_instr.funct3() == F3_0);
ASSERT(jr_instr.rs1() == FAR_TMP);
intptr_t old_offset = auipc_instr.utype_imm() + jr_instr.itype_imm();
intx_t lo = ImmLo(new_offset);
intx_t hi = ImmHi(new_offset);
if (!IsUTypeImm(hi)) {
FATAL("Jump/branch distance exceeds 2GB!");
}
Write32(branch_position,
EncodeUTypeImm(hi) | EncodeRd(FAR_TMP) | EncodeOpcode(AUIPC));
Write32(branch_position + 4, EncodeITypeImm(lo) | EncodeRs1(FAR_TMP) |
EncodeFunct3(F3_0) | EncodeRd(ZR) |
EncodeOpcode(JALR));
return old_offset;
}
void MicroAssembler::lui(Register rd, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) && (rd != ZR) && (rd != SP) && IsCUImm(imm)) {
c_lui(rd, imm);
return;
}
EmitUType(imm, rd, LUI);
}
void MicroAssembler::lui_fixed(Register rd, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitUType(imm, rd, LUI);
}
void MicroAssembler::auipc(Register rd, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitUType(imm, rd, AUIPC);
}
void MicroAssembler::jal(Register rd, Label* label, JumpDistance distance) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) &&
((distance == kNearJump) ||
(label->IsBound() && IsCJImm(label->Position() - Position())))) {
if (rd == ZR) {
c_j(label);
return;
}
#if XLEN == 32
if (rd == RA) {
c_jal(label);
return;
}
#endif // XLEN == 32
}
EmitJump(rd, label, JAL, distance);
}
void MicroAssembler::jalr(Register rd, Register rs1, intptr_t offset) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if (rs1 != ZR && offset == 0) {
if (rd == ZR) {
c_jr(rs1);
return;
} else if (rd == RA) {
c_jalr(rs1);
return;
}
}
}
EmitIType(offset, rs1, F3_0, rd, JALR);
}
void MicroAssembler::jalr_fixed(Register rd, Register rs1, intptr_t offset) {
ASSERT(Supports(RV_I));
EmitIType(offset, rs1, F3_0, rd, JALR);
}
void MicroAssembler::beq(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) &&
((distance == kNearJump) ||
(label->IsBound() && IsCBImm(label->Position() - Position())))) {
if ((rs1 == ZR) && IsCRs1p(rs2)) {
c_beqz(rs2, label);
return;
} else if ((rs2 == ZR) && IsCRs1p(rs1)) {
c_beqz(rs1, label);
return;
}
}
EmitBranch(rs1, rs2, label, BEQ, distance);
}
void MicroAssembler::bne(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
if (Supports(RV_C) &&
((distance == kNearJump) ||
(label->IsBound() && IsCBImm(label->Position() - Position())))) {
if ((rs1 == ZR) && IsCRs1p(rs2)) {
c_bnez(rs2, label);
return;
} else if ((rs2 == ZR) && IsCRs1p(rs1)) {
c_bnez(rs1, label);
return;
}
}
EmitBranch(rs1, rs2, label, BNE, distance);
}
void MicroAssembler::blt(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
EmitBranch(rs1, rs2, label, BLT, distance);
}
void MicroAssembler::bge(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
EmitBranch(rs1, rs2, label, BGE, distance);
}
void MicroAssembler::bltu(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
ASSERT(Supports(RV_I));
EmitBranch(rs1, rs2, label, BLTU, distance);
}
void MicroAssembler::bgeu(Register rs1,
Register rs2,
Label* label,
JumpDistance distance) {
EmitBranch(rs1, rs2, label, BGEU, distance);
}
void MicroAssembler::lb(Register rd, Address addr) {
ASSERT(Supports(RV_I));
EmitIType(addr.offset(), addr.base(), LB, rd, LOAD);
}
void MicroAssembler::lh(Register rd, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_Zcb) && IsCRdp(rd) && IsCRs1p(addr.base()) &&
IsCMem2Imm(addr.offset())) {
c_lh(rd, addr);
return;
}
EmitIType(addr.offset(), addr.base(), LH, rd, LOAD);
}
void MicroAssembler::lw(Register rd, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (addr.base() == SP) && IsCSPLoad4Imm(addr.offset())) {
c_lwsp(rd, addr);
return;
}
if (IsCRdp(rd) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_lw(rd, addr);
return;
}
}
EmitIType(addr.offset(), addr.base(), LW, rd, LOAD);
}
void MicroAssembler::lbu(Register rd, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_Zcb) && IsCRdp(rd) && IsCRs1p(addr.base()) &&
IsCMem1Imm(addr.offset())) {
c_lbu(rd, addr);
return;
}
EmitIType(addr.offset(), addr.base(), LBU, rd, LOAD);
}
void MicroAssembler::lhu(Register rd, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_Zcb) && IsCRdp(rd) && IsCRs1p(addr.base()) &&
IsCMem2Imm(addr.offset())) {
c_lhu(rd, addr);
return;
}
EmitIType(addr.offset(), addr.base(), LHU, rd, LOAD);
}
void MicroAssembler::sb(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_Zcb) && IsCRs2p(rs2) && IsCRs1p(addr.base()) &&
IsCMem1Imm(addr.offset())) {
c_sb(rs2, addr);
return;
}
EmitSType(addr.offset(), rs2, addr.base(), SB, STORE);
}
void MicroAssembler::sh(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_Zcb) && IsCRs2p(rs2) && IsCRs1p(addr.base()) &&
IsCMem2Imm(addr.offset())) {
c_sh(rs2, addr);
return;
}
EmitSType(addr.offset(), rs2, addr.base(), SH, STORE);
}
void MicroAssembler::sw(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore4Imm(addr.offset())) {
c_swsp(rs2, addr);
return;
}
if (IsCRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_sw(rs2, addr);
return;
}
}
EmitSType(addr.offset(), rs2, addr.base(), SW, STORE);
}
void MicroAssembler::addi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (rs1 == ZR) && IsCIImm(imm)) {
c_li(rd, imm);
return;
}
if ((rd == rs1) && IsCIImm(imm) && (imm != 0)) {
c_addi(rd, rs1, imm);
return;
}
if ((rd == SP) && (rs1 == SP) && IsCI16Imm(imm) && (imm != 0)) {
c_addi16sp(rd, rs1, imm);
return;
}
if (IsCRdp(rd) && (rs1 == SP) && IsCI4SPNImm(imm) && (imm != 0)) {
c_addi4spn(rd, rs1, imm);
return;
}
if (imm == 0) {
if ((rd == ZR) && (rs1 == ZR)) {
c_nop();
return;
}
if ((rd != ZR) && (rs1 != ZR)) {
c_mv(rd, rs1);
return;
}
}
}
EmitIType(imm, rs1, ADDI, rd, OPIMM);
}
void MicroAssembler::slti(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitIType(imm, rs1, SLTI, rd, OPIMM);
}
void MicroAssembler::sltiu(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitIType(imm, rs1, SLTIU, rd, OPIMM);
}
void MicroAssembler::xori(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_Zcb)) {
if ((rd == rs1) && IsCRs1p(rs1) && (imm == -1)) {
c_not(rd, rs1);
return;
}
}
EmitIType(imm, rs1, XORI, rd, OPIMM);
}
void MicroAssembler::ori(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
EmitIType(imm, rs1, ORI, rd, OPIMM);
}
void MicroAssembler::andi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCIImm(imm)) {
c_andi(rd, rs1, imm);
return;
}
if (Supports(RV_Zcb)) {
if ((rd == rs1) && IsCRs1p(rs1) && (imm == 0xFF)) {
c_zextb(rd, rs1);
return;
}
}
}
EmitIType(imm, rs1, ANDI, rd, OPIMM);
}
void MicroAssembler::slli(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && (rd != ZR) && IsCShamt(shamt)) {
c_slli(rd, rs1, shamt);
return;
}
}
EmitRType(F7_0, shamt, rs1, SLLI, rd, OPIMM);
}
void MicroAssembler::srli(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCShamt(shamt)) {
c_srli(rd, rs1, shamt);
return;
}
}
EmitRType(F7_0, shamt, rs1, SRI, rd, OPIMM);
}
void MicroAssembler::srai(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCShamt(shamt)) {
c_srai(rd, rs1, shamt);
return;
}
}
EmitRType(SRA, shamt, rs1, SRI, rd, OPIMM);
}
void MicroAssembler::add(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if (rd == rs1) {
c_add(rd, rs1, rs2);
return;
}
if (rd == rs2) {
c_add(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, ADD, rd, OP);
}
void MicroAssembler::sub(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_sub(rd, rs1, rs2);
return;
}
}
EmitRType(SUB, rs2, rs1, ADD, rd, OP);
}
void MicroAssembler::sll(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLL, rd, OP);
}
void MicroAssembler::slt(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLT, rd, OP);
}
void MicroAssembler::sltu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLTU, rd, OP);
}
void MicroAssembler::xor_(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_xor(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_xor(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, XOR, rd, OP);
}
void MicroAssembler::srl(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SR, rd, OP);
}
void MicroAssembler::sra(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(SRA, rs2, rs1, SR, rd, OP);
}
void MicroAssembler::or_(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_or(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_or(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, OR, rd, OP);
}
void MicroAssembler::and_(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_and(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_and(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, AND, rd, OP);
}
void MicroAssembler::fence(HartEffects predecessor, HartEffects successor) {
ASSERT((predecessor & kAll) == predecessor);
ASSERT((successor & kAll) == successor);
ASSERT(Supports(RV_I));
EmitIType((predecessor << 4) | successor, ZR, FENCE, ZR, MISCMEM);
}
void MicroAssembler::fencei() {
ASSERT(Supports(RV_I));
EmitIType(0, ZR, FENCEI, ZR, MISCMEM);
}
void MicroAssembler::ecall() {
ASSERT(Supports(RV_I));
EmitIType(ECALL, ZR, F3_0, ZR, SYSTEM);
}
void MicroAssembler::ebreak() {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
c_ebreak();
return;
}
EmitIType(EBREAK, ZR, F3_0, ZR, SYSTEM);
}
void MicroAssembler::SimulatorPrintObject(Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(ECALL, rs1, F3_0, ZR, SYSTEM);
}
void MicroAssembler::csrrw(Register rd, uint32_t csr, Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(csr, rs1, CSRRW, rd, SYSTEM);
}
void MicroAssembler::csrrs(Register rd, uint32_t csr, Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(csr, rs1, CSRRS, rd, SYSTEM);
}
void MicroAssembler::csrrc(Register rd, uint32_t csr, Register rs1) {
ASSERT(Supports(RV_I));
EmitIType(csr, rs1, CSRRC, rd, SYSTEM);
}
void MicroAssembler::csrrwi(Register rd, uint32_t csr, uint32_t imm) {
ASSERT(Supports(RV_I));
EmitIType(csr, Register(imm), CSRRWI, rd, SYSTEM);
}
void MicroAssembler::csrrsi(Register rd, uint32_t csr, uint32_t imm) {
ASSERT(Supports(RV_I));
EmitIType(csr, Register(imm), CSRRSI, rd, SYSTEM);
}
void MicroAssembler::csrrci(Register rd, uint32_t csr, uint32_t imm) {
ASSERT(Supports(RV_I));
EmitIType(csr, Register(imm), CSRRCI, rd, SYSTEM);
}
void MicroAssembler::trap() {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
Emit16(0); // Permanently reserved illegal instruction.
} else {
Emit32(0); // Permanently reserved illegal instruction.
}
}
#if XLEN >= 64
void MicroAssembler::lwu(Register rd, Address addr) {
ASSERT(Supports(RV_I));
EmitIType(addr.offset(), addr.base(), LWU, rd, LOAD);
}
void MicroAssembler::ld(Register rd, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (addr.base() == SP) && IsCSPLoad8Imm(addr.offset())) {
c_ldsp(rd, addr);
return;
}
if (IsCRdp(rd) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_ld(rd, addr);
return;
}
}
EmitIType(addr.offset(), addr.base(), LD, rd, LOAD);
}
void MicroAssembler::sd(Register rs2, Address addr) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore8Imm(addr.offset())) {
c_sdsp(rs2, addr);
return;
}
if (IsCRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_sd(rs2, addr);
return;
}
}
EmitSType(addr.offset(), rs2, addr.base(), SD, STORE);
}
void MicroAssembler::addiw(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd != ZR) && (rs1 == ZR) && IsCIImm(imm)) {
c_li(rd, imm);
return;
}
if ((rd == rs1) && (rd != ZR) && IsCIImm(imm)) {
c_addiw(rd, rs1, imm);
return;
}
}
EmitIType(imm, rs1, ADDI, rd, OPIMM32);
}
void MicroAssembler::slliw(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < 32));
ASSERT(Supports(RV_I));
EmitRType(F7_0, shamt, rs1, SLLI, rd, OPIMM32);
}
void MicroAssembler::srliw(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < 32));
ASSERT(Supports(RV_I));
EmitRType(F7_0, shamt, rs1, SRI, rd, OPIMM32);
}
void MicroAssembler::sraiw(Register rd, Register rs1, intptr_t shamt) {
ASSERT((shamt > 0) && (shamt < XLEN));
ASSERT(Supports(RV_I));
EmitRType(SRA, shamt, rs1, SRI, rd, OPIMM32);
}
void MicroAssembler::addw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_addw(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_addw(rd, rs2, rs1);
return;
}
}
EmitRType(F7_0, rs2, rs1, ADD, rd, OP32);
}
void MicroAssembler::subw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
if (Supports(RV_C)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_subw(rd, rs1, rs2);
return;
}
}
EmitRType(SUB, rs2, rs1, ADD, rd, OP32);
}
void MicroAssembler::sllw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SLL, rd, OP32);
}
void MicroAssembler::srlw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(F7_0, rs2, rs1, SR, rd, OP32);
}
void MicroAssembler::sraw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_I));
EmitRType(SRA, rs2, rs1, SR, rd, OP32);
}
#endif // XLEN >= 64
void MicroAssembler::mul(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
if (Supports(RV_Zcb)) {
if ((rd == rs1) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_mul(rd, rs1, rs2);
return;
}
if ((rd == rs2) && IsCRs1p(rs1) && IsCRs2p(rs2)) {
c_mul(rd, rs2, rs1);
return;
}
}
EmitRType(MULDIV, rs2, rs1, MUL, rd, OP);
}
void MicroAssembler::mulh(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULH, rd, OP);
}
void MicroAssembler::mulhsu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULHSU, rd, OP);
}
void MicroAssembler::mulhu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULHU, rd, OP);
}
void MicroAssembler::div(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIV, rd, OP);
}
void MicroAssembler::divu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIVU, rd, OP);
}
void MicroAssembler::rem(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REM, rd, OP);
}
void MicroAssembler::remu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REMU, rd, OP);
}
#if XLEN >= 64
void MicroAssembler::mulw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, MULW, rd, OP32);
}
void MicroAssembler::divw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIVW, rd, OP32);
}
void MicroAssembler::divuw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, DIVUW, rd, OP32);
}
void MicroAssembler::remw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REMW, rd, OP32);
}
void MicroAssembler::remuw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_M));
EmitRType(MULDIV, rs2, rs1, REMUW, rd, OP32);
}
#endif // XLEN >= 64
void MicroAssembler::lrw(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(LR, order, ZR, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::scw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(SC, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoswapw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOSWAP, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoaddw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOADD, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoxorw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOXOR, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoandw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOAND, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amoorw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOOR, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amominw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMIN, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amomaxw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAX, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amominuw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMINU, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::amomaxuw(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAXU, order, rs2, addr.base(), WIDTH32, rd, AMO);
}
#if XLEN >= 64
void MicroAssembler::lrd(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(LR, order, ZR, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::scd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(SC, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoswapd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOSWAP, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoaddd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOADD, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoxord(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOXOR, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoandd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOAND, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amoord(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOOR, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amomind(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMIN, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amomaxd(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAX, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amominud(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMINU, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::amomaxud(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_A));
EmitRType(AMOMAXU, order, rs2, addr.base(), WIDTH64, rd, AMO);
}
#endif // XLEN >= 64
void MicroAssembler::flw(FRegister rd, Address addr) {
ASSERT(Supports(RV_F));
#if XLEN == 32
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPLoad4Imm(addr.offset())) {
c_flwsp(rd, addr);
return;
}
if (IsCFRdp(rd) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_flw(rd, addr);
return;
}
}
#endif // XLEN == 32
EmitIType(addr.offset(), addr.base(), S, rd, LOADFP);
}
void MicroAssembler::fsw(FRegister rs2, Address addr) {
ASSERT(Supports(RV_F));
#if XLEN == 32
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore4Imm(addr.offset())) {
c_fswsp(rs2, addr);
return;
}
if (IsCFRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem4Imm(addr.offset())) {
c_fsw(rs2, addr);
return;
}
}
#endif // XLEN == 32
EmitSType(addr.offset(), rs2, addr.base(), S, STOREFP);
}
void MicroAssembler::fmadds(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FMADD);
}
void MicroAssembler::fmsubs(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FMSUB);
}
void MicroAssembler::fnmsubs(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FNMSUB);
}
void MicroAssembler::fnmadds(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitR4Type(rs3, F2_S, rs2, rs1, rounding, rd, FNMADD);
}
void MicroAssembler::fadds(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FADDS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsubs(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FSUBS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmuls(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FMULS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fdivs(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FDIVS, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsqrts(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FSQRTS, FRegister(0), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsgnjs(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FSGNJS, rs2, rs1, J, rd, OPFP);
}
void MicroAssembler::fsgnjns(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FSGNJS, rs2, rs1, JN, rd, OPFP);
}
void MicroAssembler::fsgnjxs(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FSGNJS, rs2, rs1, JX, rd, OPFP);
}
void MicroAssembler::fmins(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FMINMAXS, rs2, rs1, FMIN, rd, OPFP);
}
void MicroAssembler::fmaxs(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FMINMAXS, rs2, rs1, FMAX, rd, OPFP);
}
void MicroAssembler::feqs(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FCMPS, rs2, rs1, FEQ, rd, OPFP);
}
void MicroAssembler::flts(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FCMPS, rs2, rs1, FLT, rd, OPFP);
}
void MicroAssembler::fles(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_F));
EmitRType(FCMPS, rs2, rs1, FLE, rd, OPFP);
}
void MicroAssembler::fclasss(Register rd, FRegister rs1) {
ASSERT(Supports(RV_F));
EmitRType(FCLASSS, FRegister(0), rs1, F3_1, rd, OPFP);
}
void MicroAssembler::fcvtws(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtwus(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(WU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtsw(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, Register(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtswu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, Register(WU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmvxw(Register rd, FRegister rs1) {
ASSERT(Supports(RV_F));
EmitRType(FMVXW, FRegister(0), rs1, F3_0, rd, OPFP);
}
void MicroAssembler::fmvwx(FRegister rd, Register rs1) {
ASSERT(Supports(RV_F));
EmitRType(FMVWX, Register(0), rs1, F3_0, rd, OPFP);
}
#if XLEN >= 64
void MicroAssembler::fcvtls(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtlus(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTintS, FRegister(LU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtsl(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, Register(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtslu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_F));
EmitRType(FCVTSint, Register(LU), rs1, rounding, rd, OPFP);
}
#endif // XLEN >= 64
void MicroAssembler::fld(FRegister rd, Address addr) {
ASSERT(Supports(RV_D));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPLoad8Imm(addr.offset())) {
c_fldsp(rd, addr);
return;
}
if (IsCFRdp(rd) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_fld(rd, addr);
return;
}
}
EmitIType(addr.offset(), addr.base(), D, rd, LOADFP);
}
void MicroAssembler::fsd(FRegister rs2, Address addr) {
ASSERT(Supports(RV_D));
if (Supports(RV_C)) {
if ((addr.base() == SP) && IsCSPStore8Imm(addr.offset())) {
c_fsdsp(rs2, addr);
return;
}
if (IsCFRs2p(rs2) && IsCRs1p(addr.base()) && IsCMem8Imm(addr.offset())) {
c_fsd(rs2, addr);
return;
}
}
EmitSType(addr.offset(), rs2, addr.base(), D, STOREFP);
}
void MicroAssembler::fmaddd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FMADD);
}
void MicroAssembler::fmsubd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FMSUB);
}
void MicroAssembler::fnmsubd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FNMSUB);
}
void MicroAssembler::fnmaddd(FRegister rd,
FRegister rs1,
FRegister rs2,
FRegister rs3,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitR4Type(rs3, F2_D, rs2, rs1, rounding, rd, FNMADD);
}
void MicroAssembler::faddd(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FADDD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsubd(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FSUBD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmuld(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FMULD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fdivd(FRegister rd,
FRegister rs1,
FRegister rs2,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FDIVD, rs2, rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsqrtd(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FSQRTD, FRegister(0), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fsgnjd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FSGNJD, rs2, rs1, J, rd, OPFP);
}
void MicroAssembler::fsgnjnd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FSGNJD, rs2, rs1, JN, rd, OPFP);
}
void MicroAssembler::fsgnjxd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FSGNJD, rs2, rs1, JX, rd, OPFP);
}
void MicroAssembler::fmind(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FMINMAXD, rs2, rs1, FMIN, rd, OPFP);
}
void MicroAssembler::fmaxd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FMINMAXD, rs2, rs1, FMAX, rd, OPFP);
}
void MicroAssembler::fcvtsd(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTS, FRegister(1), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtds(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTD, FRegister(0), rs1, rounding, rd, OPFP);
}
void MicroAssembler::feqd(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FCMPD, rs2, rs1, FEQ, rd, OPFP);
}
void MicroAssembler::fltd(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FCMPD, rs2, rs1, FLT, rd, OPFP);
}
void MicroAssembler::fled(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_D));
EmitRType(FCMPD, rs2, rs1, FLE, rd, OPFP);
}
void MicroAssembler::fclassd(Register rd, FRegister rs1) {
ASSERT(Supports(RV_D));
EmitRType(FCLASSD, FRegister(0), rs1, F3_1, rd, OPFP);
}
void MicroAssembler::fcvtwd(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtwud(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(WU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtdw(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, Register(W), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtdwu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, Register(WU), rs1, rounding, rd, OPFP);
}
#if XLEN >= 64
void MicroAssembler::fcvtld(Register rd, FRegister rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtlud(Register rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTintD, FRegister(LU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmvxd(Register rd, FRegister rs1) {
ASSERT(Supports(RV_D));
EmitRType(FMVXD, FRegister(0), rs1, F3_0, rd, OPFP);
}
void MicroAssembler::fcvtdl(FRegister rd, Register rs1, RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, Register(L), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtdlu(FRegister rd,
Register rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_D));
EmitRType(FCVTDint, Register(LU), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fmvdx(FRegister rd, Register rs1) {
ASSERT(Supports(RV_D));
EmitRType(FMVDX, Register(0), rs1, F3_0, rd, OPFP);
}
#endif // XLEN >= 64
#if XLEN >= 64
void MicroAssembler::adduw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zba));
if (Supports(RV_Zcb)) {
if ((rd == rs1) && IsCRs1p(rs1) && (rs2 == ZR)) {
return c_zextw(rd, rs1);
}
}
EmitRType(ADDUW, rs2, rs1, F3_0, rd, OP32);
}
#endif
void MicroAssembler::sh1add(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zba));
EmitRType(SHADD, rs2, rs1, SH1ADD, rd, OP);
}
#if XLEN >= 64
void MicroAssembler::sh1adduw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zba));
EmitRType(SHADD, rs2, rs1, SH1ADD, rd, OP32);
}
#endif
void MicroAssembler::sh2add(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zba));
EmitRType(SHADD, rs2, rs1, SH2ADD, rd, OP);
}
#if XLEN >= 64
void MicroAssembler::sh2adduw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zba));
EmitRType(SHADD, rs2, rs1, SH2ADD, rd, OP32);
}
#endif
void MicroAssembler::sh3add(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zba));
EmitRType(SHADD, rs2, rs1, SH3ADD, rd, OP);
}
#if XLEN >= 64
void MicroAssembler::sh3adduw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zba));
EmitRType(SHADD, rs2, rs1, SH3ADD, rd, OP32);
}
void MicroAssembler::slliuw(Register rd, Register rs1, intx_t shamt) {
ASSERT((shamt > 0) && (shamt < 32));
ASSERT(Supports(RV_Zba));
EmitRType(SLLIUW, shamt, rs1, SLLI, rd, OPIMM32);
}
#endif
void MicroAssembler::andn(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(SUB, rs2, rs1, AND, rd, OP);
}
void MicroAssembler::orn(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(SUB, rs2, rs1, OR, rd, OP);
}
void MicroAssembler::xnor(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(SUB, rs2, rs1, XOR, rd, OP);
}
void MicroAssembler::clz(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
EmitRType(COUNT, 0b00000, rs1, F3_COUNT, rd, OPIMM);
}
void MicroAssembler::clzw(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
EmitRType(COUNT, 0b00000, rs1, F3_COUNT, rd, OPIMM32);
}
void MicroAssembler::ctz(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
EmitRType(COUNT, 0b00001, rs1, F3_COUNT, rd, OPIMM);
}
void MicroAssembler::ctzw(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
EmitRType(COUNT, 0b00001, rs1, F3_COUNT, rd, OPIMM32);
}
void MicroAssembler::cpop(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
EmitRType(COUNT, 0b00010, rs1, F3_COUNT, rd, OPIMM);
}
void MicroAssembler::cpopw(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
EmitRType(COUNT, 0b00010, rs1, F3_COUNT, rd, OPIMM32);
}
void MicroAssembler::max(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(MINMAXCLMUL, rs2, rs1, MAX, rd, OP);
}
void MicroAssembler::maxu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(MINMAXCLMUL, rs2, rs1, MAXU, rd, OP);
}
void MicroAssembler::min(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(MINMAXCLMUL, rs2, rs1, MIN, rd, OP);
}
void MicroAssembler::minu(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(MINMAXCLMUL, rs2, rs1, MINU, rd, OP);
}
void MicroAssembler::sextb(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
if (Supports(RV_Zcb)) {
if ((rd == rs1) && IsCRs1p(rs1)) {
return c_sextb(rd, rs1);
}
}
EmitRType((Funct7)0b0110000, 0b00100, rs1, SEXT, rd, OPIMM);
}
void MicroAssembler::sexth(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
if (Supports(RV_Zcb)) {
if ((rd == rs1) && IsCRs1p(rs1)) {
return c_sexth(rd, rs1);
}
}
EmitRType((Funct7)0b0110000, 0b00101, rs1, SEXT, rd, OPIMM);
}
void MicroAssembler::zexth(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
if (Supports(RV_Zcb)) {
if ((rd == rs1) && IsCRs1p(rs1)) {
return c_zexth(rd, rs1);
}
}
#if XLEN == 32
EmitRType((Funct7)0b0000100, 0b00000, rs1, ZEXT, rd, OP);
#elif XLEN == 64
EmitRType((Funct7)0b0000100, 0b00000, rs1, ZEXT, rd, OP32);
#else
UNIMPLEMENTED();
#endif
}
void MicroAssembler::rol(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(ROTATE, rs2, rs1, ROL, rd, OP);
}
void MicroAssembler::rolw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(ROTATE, rs2, rs1, ROL, rd, OP32);
}
void MicroAssembler::ror(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(ROTATE, rs2, rs1, ROR, rd, OP);
}
void MicroAssembler::rori(Register rd, Register rs1, intx_t shamt) {
ASSERT(Supports(RV_Zbb));
EmitRType(ROTATE, shamt, rs1, ROR, rd, OPIMM);
}
void MicroAssembler::roriw(Register rd, Register rs1, intx_t shamt) {
ASSERT(Supports(RV_Zbb));
EmitRType(ROTATE, shamt, rs1, ROR, rd, OPIMM32);
}
void MicroAssembler::rorw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbb));
EmitRType(ROTATE, rs2, rs1, ROR, rd, OP32);
}
void MicroAssembler::orcb(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
EmitRType((Funct7)0b0010100, 0b00111, rs1, (Funct3)0b101, rd, OPIMM);
}
void MicroAssembler::rev8(Register rd, Register rs1) {
ASSERT(Supports(RV_Zbb));
#if XLEN == 32
EmitRType((Funct7)0b0110100, 0b11000, rs1, (Funct3)0b101, rd, OPIMM);
#elif XLEN == 64
EmitRType((Funct7)0b0110101, 0b11000, rs1, (Funct3)0b101, rd, OPIMM);
#else
UNIMPLEMENTED();
#endif
}
void MicroAssembler::clmul(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbc));
EmitRType(MINMAXCLMUL, rs2, rs1, CLMUL, rd, OP);
}
void MicroAssembler::clmulh(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbc));
EmitRType(MINMAXCLMUL, rs2, rs1, CLMULH, rd, OP);
}
void MicroAssembler::clmulr(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbc));
EmitRType(MINMAXCLMUL, rs2, rs1, CLMULR, rd, OP);
}
void MicroAssembler::bclr(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbs));
EmitRType(BCLRBEXT, rs2, rs1, BCLR, rd, OP);
}
void MicroAssembler::bclri(Register rd, Register rs1, intx_t shamt) {
ASSERT(Supports(RV_Zbs));
EmitRType(BCLRBEXT, shamt, rs1, BCLR, rd, OPIMM);
}
void MicroAssembler::bext(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbs));
EmitRType(BCLRBEXT, rs2, rs1, BEXT, rd, OP);
}
void MicroAssembler::bexti(Register rd, Register rs1, intx_t shamt) {
ASSERT(Supports(RV_Zbs));
EmitRType(BCLRBEXT, shamt, rs1, BEXT, rd, OPIMM);
}
void MicroAssembler::binv(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbs));
EmitRType(BINV, rs2, rs1, F3_BINV, rd, OP);
}
void MicroAssembler::binvi(Register rd, Register rs1, intx_t shamt) {
ASSERT(Supports(RV_Zbs));
EmitRType(BINV, shamt, rs1, F3_BINV, rd, OPIMM);
}
void MicroAssembler::bset(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zbs));
EmitRType(BSET, rs2, rs1, F3_BSET, rd, OP);
}
void MicroAssembler::bseti(Register rd, Register rs1, intx_t shamt) {
ASSERT(Supports(RV_Zbs));
EmitRType(BSET, shamt, rs1, F3_BSET, rd, OPIMM);
}
void MicroAssembler::czeroeqz(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zicond));
EmitRType(CZERO, rs2, rs1, CZEROEQZ, rd, OP);
}
void MicroAssembler::czeronez(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zicond));
EmitRType(CZERO, rs2, rs1, CZERONEZ, rd, OP);
}
void MicroAssembler::amoswapb(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOSWAP, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amoaddb(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOADD, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amoxorb(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOXOR, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amoandb(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOAND, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amoorb(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOOR, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amominb(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMIN, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amomaxb(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMAX, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amominub(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMINU, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amomaxub(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMAXU, order, rs2, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::amoswaph(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOSWAP, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amoaddh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOADD, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amoxorh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOXOR, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amoandh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOAND, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amoorh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOOR, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amominh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMIN, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amomaxh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMAX, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amominuh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMINU, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::amomaxuh(Register rd,
Register rs2,
Address addr,
std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT(Supports(RV_Zabha));
EmitRType(AMOMAXU, order, rs2, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::flis(FRegister rd, intptr_t index) {
ASSERT((index >= 0) && (index < 32));
ASSERT(Supports(RV_Zfa));
EmitRType(FMVWX, FRegister(1), FRegister(index), F3_0, rd, OPFP);
}
void MicroAssembler::flid(FRegister rd, intptr_t index) {
ASSERT((index >= 0) && (index < 32));
ASSERT(Supports(RV_Zfa));
EmitRType(FMVDX, FRegister(1), FRegister(index), F3_0, rd, OPFP);
}
void MicroAssembler::fminms(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FMINMAXS, rs2, rs1, FMINM, rd, OPFP);
}
void MicroAssembler::fmaxms(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FMINMAXS, rs2, rs1, FMAXM, rd, OPFP);
}
void MicroAssembler::fminmd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FMINMAXD, rs2, rs1, FMINM, rd, OPFP);
}
void MicroAssembler::fmaxmd(FRegister rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FMINMAXD, rs2, rs1, FMAXM, rd, OPFP);
}
void MicroAssembler::frounds(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCVTS, FRegister(4), rs1, rounding, rd, OPFP);
}
void MicroAssembler::froundnxs(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCVTS, FRegister(5), rs1, rounding, rd, OPFP);
}
void MicroAssembler::froundd(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCVTD, FRegister(4), rs1, rounding, rd, OPFP);
}
void MicroAssembler::froundnxd(FRegister rd,
FRegister rs1,
RoundingMode rounding) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCVTD, FRegister(5), rs1, rounding, rd, OPFP);
}
void MicroAssembler::fcvtmodwd(Register rd, FRegister rs1) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCVTintD, FRegister(8), rs1, RTZ, rd, OPFP);
}
#if XLEN == 32
void MicroAssembler::fmvhxd(Register rd, FRegister rs1) {
ASSERT(Supports(RV_Zfa));
EmitRType(FMVHXD, FRegister(1), rs1, F3_0, rd, OPFP);
}
void MicroAssembler::fmvpdx(FRegister rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FMVPDX, rs2, rs1, F3_0, rd, OPFP);
}
#endif // XLEN == 32
void MicroAssembler::fltqs(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCMPS, rs2, rs1, FLTQ, rd, OPFP);
}
void MicroAssembler::fleqs(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCMPS, rs2, rs1, FLEQ, rd, OPFP);
}
void MicroAssembler::fltqd(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCMPD, rs2, rs1, FLTQ, rd, OPFP);
}
void MicroAssembler::fleqd(Register rd, FRegister rs1, FRegister rs2) {
ASSERT(Supports(RV_Zfa));
EmitRType(FCMPD, rs2, rs1, FLEQ, rd, OPFP);
}
void MicroAssembler::lb(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_acquire) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH8, rd, AMO);
}
void MicroAssembler::lh(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_acquire) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH16, rd, AMO);
}
void MicroAssembler::lw(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_acquire) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH32, rd, AMO);
}
void MicroAssembler::sb(Register rs2, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_release) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH8, ZR, AMO);
}
void MicroAssembler::sh(Register rs2, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_release) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH16, ZR, AMO);
}
void MicroAssembler::sw(Register rs2, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_release) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH32, ZR, AMO);
}
#if XLEN >= 64
void MicroAssembler::ld(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_acquire) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH64, rd, AMO);
}
void MicroAssembler::sd(Register rs2, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_release) ||
(order == std::memory_order_acq_rel));
ASSERT(Supports(RV_Zalasr));
EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH64, ZR, AMO);
}
#endif
void MicroAssembler::c_lwsp(Register rd, Address addr) {
ASSERT(rd != ZR);
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_LWSP | EncodeCRd(rd) | EncodeCSPLoad4Imm(addr.offset()));
}
#if XLEN == 32
void MicroAssembler::c_flwsp(FRegister rd, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FLWSP | EncodeCFRd(rd) | EncodeCSPLoad4Imm(addr.offset()));
}
#else
void MicroAssembler::c_ldsp(Register rd, Address addr) {
ASSERT(rd != ZR);
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_LDSP | EncodeCRd(rd) | EncodeCSPLoad8Imm(addr.offset()));
}
#endif
void MicroAssembler::c_fldsp(FRegister rd, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FLDSP | EncodeCFRd(rd) | EncodeCSPLoad8Imm(addr.offset()));
}
void MicroAssembler::c_swsp(Register rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_SWSP | EncodeCRs2(rs2) | EncodeCSPStore4Imm(addr.offset()));
}
#if XLEN == 32
void MicroAssembler::c_fswsp(FRegister rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FSWSP | EncodeCFRs2(rs2) | EncodeCSPStore4Imm(addr.offset()));
}
#else
void MicroAssembler::c_sdsp(Register rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
Emit16(C_SDSP | EncodeCRs2(rs2) | EncodeCSPStore8Imm(addr.offset()));
}
#endif
void MicroAssembler::c_fsdsp(FRegister rs2, Address addr) {
ASSERT(addr.base() == SP);
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FSDSP | EncodeCFRs2(rs2) | EncodeCSPStore8Imm(addr.offset()));
}
void MicroAssembler::c_lw(Register rd, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_LW | EncodeCRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_ld(Register rd, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_LD | EncodeCRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_flw(FRegister rd, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FLW | EncodeCFRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_fld(FRegister rd, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FLD | EncodeCFRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_sw(Register rs2, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_SW | EncodeCRs1p(addr.base()) | EncodeCRs2p(rs2) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_sd(Register rs2, Address addr) {
ASSERT(Supports(RV_C));
Emit16(C_SD | EncodeCRs1p(addr.base()) | EncodeCRs2p(rs2) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_fsw(FRegister rs2, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_F));
Emit16(C_FSW | EncodeCRs1p(addr.base()) | EncodeCFRs2p(rs2) |
EncodeCMem4Imm(addr.offset()));
}
void MicroAssembler::c_fsd(FRegister rs2, Address addr) {
ASSERT(Supports(RV_C));
ASSERT(Supports(RV_D));
Emit16(C_FSD | EncodeCRs1p(addr.base()) | EncodeCFRs2p(rs2) |
EncodeCMem8Imm(addr.offset()));
}
void MicroAssembler::c_j(Label* label) {
ASSERT(Supports(RV_C));
EmitCJump(label, C_J);
}
#if XLEN == 32
void MicroAssembler::c_jal(Label* label) {
ASSERT(Supports(RV_C));
EmitCJump(label, C_JAL);
}
#endif // XLEN == 32
void MicroAssembler::c_jr(Register rs1) {
ASSERT(Supports(RV_C));
ASSERT(rs1 != ZR);
Emit16(C_JR | EncodeCRs1(rs1) | EncodeCRs2(ZR));
}
void MicroAssembler::c_jalr(Register rs1) {
ASSERT(Supports(RV_C));
Emit16(C_JALR | EncodeCRs1(rs1) | EncodeCRs2(ZR));
}
void MicroAssembler::c_beqz(Register rs1p, Label* label) {
ASSERT(Supports(RV_C));
EmitCBranch(rs1p, label, C_BEQZ);
}
void MicroAssembler::c_bnez(Register rs1p, Label* label) {
ASSERT(Supports(RV_C));
EmitCBranch(rs1p, label, C_BNEZ);
}
void MicroAssembler::c_li(Register rd, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
Emit16(C_LI | EncodeCRd(rd) | EncodeCIImm(imm));
}
void MicroAssembler::c_lui(Register rd, uintptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
ASSERT(rd != SP);
Emit16(C_LUI | EncodeCRd(rd) | EncodeCUImm(imm));
}
void MicroAssembler::c_addi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(imm != 0);
ASSERT(rd == rs1);
Emit16(C_ADDI | EncodeCRd(rd) | EncodeCIImm(imm));
}
#if XLEN >= 64
void MicroAssembler::c_addiw(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_ADDIW | EncodeCRd(rd) | EncodeCIImm(imm));
}
#endif
void MicroAssembler::c_addi16sp(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_ADDI16SP | EncodeCRd(rd) | EncodeCI16Imm(imm));
}
void MicroAssembler::c_addi4spn(Register rdp, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rs1 == SP);
ASSERT(imm != 0);
Emit16(C_ADDI4SPN | EncodeCRdp(rdp) | EncodeCI4SPNImm(imm));
}
void MicroAssembler::c_slli(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
ASSERT(rd != ZR);
Emit16(C_SLLI | EncodeCRd(rd) | EncodeCShamt(imm));
}
void MicroAssembler::c_srli(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_SRLI | EncodeCRs1p(rd) | EncodeCShamt(imm));
}
void MicroAssembler::c_srai(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_SRAI | EncodeCRs1p(rd) | EncodeCShamt(imm));
}
void MicroAssembler::c_andi(Register rd, Register rs1, intptr_t imm) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_ANDI | EncodeCRs1p(rd) | EncodeCIImm(imm));
}
void MicroAssembler::c_mv(Register rd, Register rs2) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
ASSERT(rs2 != ZR);
Emit16(C_MV | EncodeCRd(rd) | EncodeCRs2(rs2));
}
void MicroAssembler::c_add(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
ASSERT(rd != ZR);
ASSERT(rd == rs1);
ASSERT(rs2 != ZR);
Emit16(C_ADD | EncodeCRd(rd) | EncodeCRs2(rs2));
}
void MicroAssembler::c_and(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
ASSERT(rd == rs1);
Emit16(C_AND | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_or(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_OR | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_xor(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_XOR | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_sub(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_SUB | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
#if XLEN >= 64
void MicroAssembler::c_addw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_ADDW | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_subw(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_C));
Emit16(C_SUBW | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
#endif // XLEN >= 64
void MicroAssembler::c_nop() {
ASSERT(Supports(RV_C));
Emit16(C_NOP);
}
void MicroAssembler::c_ebreak() {
ASSERT(Supports(RV_C));
Emit16(C_EBREAK);
}
void MicroAssembler::c_lbu(Register rd, Address addr) {
ASSERT(Supports(RV_Zcb));
Emit16(C_LBU | EncodeCRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem1Imm(addr.offset()));
}
void MicroAssembler::c_lhu(Register rd, Address addr) {
ASSERT(Supports(RV_Zcb));
Emit16(C_LHU | EncodeCRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem2Imm(addr.offset()));
}
void MicroAssembler::c_lh(Register rd, Address addr) {
ASSERT(Supports(RV_Zcb));
Emit16(C_LH | EncodeCRdp(rd) | EncodeCRs1p(addr.base()) |
EncodeCMem2Imm(addr.offset()));
}
void MicroAssembler::c_sb(Register rs2, Address addr) {
ASSERT(Supports(RV_Zcb));
Emit16(C_SB | EncodeCRs1p(addr.base()) | EncodeCRs2p(rs2) |
EncodeCMem1Imm(addr.offset()));
}
void MicroAssembler::c_sh(Register rs2, Address addr) {
ASSERT(Supports(RV_Zcb));
Emit16(C_SH | EncodeCRs1p(addr.base()) | EncodeCRs2p(rs2) |
EncodeCMem2Imm(addr.offset()));
}
void MicroAssembler::c_zextb(Register rd, Register rs1) {
ASSERT(Supports(RV_Zcb));
ASSERT(rd == rs1);
Emit16(C_ZEXTB | EncodeCRs1p(rs1));
}
void MicroAssembler::c_sextb(Register rd, Register rs1) {
ASSERT(Supports(RV_Zcb));
ASSERT(rd == rs1);
Emit16(C_SEXTB | EncodeCRs1p(rs1));
}
void MicroAssembler::c_zexth(Register rd, Register rs1) {
ASSERT(Supports(RV_Zcb));
ASSERT(Supports(RV_Zbb));
ASSERT(rd == rs1);
Emit16(C_ZEXTH | EncodeCRs1p(rs1));
}
void MicroAssembler::c_sexth(Register rd, Register rs1) {
ASSERT(Supports(RV_Zcb));
ASSERT(Supports(RV_Zbb));
ASSERT(rd == rs1);
Emit16(C_SEXTH | EncodeCRs1p(rs1));
}
#if XLEN >= 64
void MicroAssembler::c_zextw(Register rd, Register rs1) {
ASSERT(Supports(RV_Zcb));
ASSERT(Supports(RV_Zba));
ASSERT(rd == rs1);
Emit16(C_ZEXTW | EncodeCRs1p(rs1));
}
#endif
void MicroAssembler::c_mul(Register rd, Register rs1, Register rs2) {
ASSERT(Supports(RV_Zcb));
ASSERT(Supports(RV_M));
ASSERT(rd == rs1);
Emit16(C_MUL | EncodeCRs1p(rs1) | EncodeCRs2p(rs2));
}
void MicroAssembler::c_not(Register rd, Register rs1) {
ASSERT(Supports(RV_Zcb));
ASSERT(rd == rs1);
Emit16(C_NOT | EncodeCRs1p(rs1));
}
static Funct3 InvertFunct3(Funct3 func) {
switch (func) {
case BEQ:
return BNE;
case BNE:
return BEQ;
case BGE:
return BLT;
case BGEU:
return BLTU;
case BLT:
return BGE;
case BLTU:
return BGEU;
default:
UNREACHABLE();
}
}
void MicroAssembler::EmitBranch(Register rs1,
Register rs2,
Label* label,
Funct3 func,
JumpDistance distance) {
intptr_t offset;
if (label->IsBound()) {
// Backward branch: use near or far branch based on actual distance.
offset = label->Position() - Position();
if (IsBTypeImm(offset)) {
EmitBType(offset, rs2, rs1, func, BRANCH);
return;
}
if (IsJTypeImm(offset + 4)) {
intptr_t start = Position();
const intptr_t kFarBranchLength = 8;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
offset = label->Position() - Position();
EmitJType(offset, ZR, JAL);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
return;
}
intptr_t start = Position();
const intptr_t kFarBranchLength = 12;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
offset = label->Position() - Position();
intx_t lo = ImmLo(offset);
intx_t hi = ImmHi(offset);
if (!IsUTypeImm(hi)) {
FATAL("Branch distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
return;
} else {
// Forward branch: speculatively use near branches and re-assemble with far
// branches if any need greater length.
if (distance == kNearJump) {
offset = label->link_b(Position());
if (!IsBTypeImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
EmitBType(offset, rs2, rs1, func, BRANCH);
} else if (far_branch_level() == 0) {
offset = label->link_b(Position());
if (!IsBTypeImm(offset)) {
// TODO(riscv): This isn't so much because the branch is out of range
// as some previous jump to the same target would be out of B-type
// range... A possible alternative is to have separate lists on Labels
// for pending B-type and J-type instructions.
BailoutWithBranchOffsetError();
}
EmitBType(offset, rs2, rs1, func, BRANCH);
} else if (far_branch_level() == 1) {
intptr_t start = Position();
intptr_t kFarBranchLength;
if (Supports(RV_C) && (func == BEQ || func == BNE) &&
((rs1 == ZR && IsCRs1p(rs2)) || (rs2 == ZR && IsCRs1p(rs1)))) {
kFarBranchLength = 6;
Emit16((func == BEQ ? C_BNEZ : C_BEQZ) |
EncodeCRs1p(rs1 == ZR ? rs2 : rs1) |
EncodeCBImm(kFarBranchLength));
} else {
kFarBranchLength = 8;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
}
offset = label->link_j(Position());
EmitJType(offset, ZR, JAL);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
} else {
intptr_t start = Position();
intptr_t kFarBranchLength;
if (Supports(RV_C) && (func == BEQ || func == BNE) &&
((rs1 == ZR && IsCRs1p(rs2)) || (rs2 == ZR && IsCRs1p(rs1)))) {
kFarBranchLength = 10;
Emit16((func == BEQ ? C_BNEZ : C_BEQZ) |
EncodeCRs1p(rs1 == ZR ? rs2 : rs1) |
EncodeCBImm(kFarBranchLength));
} else {
kFarBranchLength = 12;
EmitBType(kFarBranchLength, rs2, rs1, InvertFunct3(func), BRANCH);
}
offset = label->link_far(Position());
intx_t lo = ImmLo(offset);
intx_t hi = ImmHi(offset);
if (!IsUTypeImm(hi)) {
FATAL("Branch distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
intptr_t end = Position();
ASSERT_EQUAL(end - start, kFarBranchLength);
}
}
}
void MicroAssembler::EmitJump(Register rd,
Label* label,
Opcode op,
JumpDistance distance) {
intptr_t offset;
if (label->IsBound()) {
// Backward jump: use near or far jump based on actual distance.
offset = label->Position() - Position();
if (IsJTypeImm(offset)) {
EmitJType(offset, rd, JAL);
return;
}
intx_t lo = ImmLo(offset);
intx_t hi = ImmHi(offset);
if (!IsUTypeImm(hi)) {
FATAL("Jump distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
return;
} else {
// Forward jump: speculatively use near jumps and re-assemble with far
// jumps if any need greater length.
if (distance == kNearJump) {
offset = label->link_j(Position());
if (!IsJTypeImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
EmitJType(offset, rd, JAL);
} else if (far_branch_level() < 2) {
offset = label->link_j(Position());
if (!IsJTypeImm(offset)) {
BailoutWithBranchOffsetError();
}
EmitJType(offset, rd, JAL);
} else {
offset = label->link_far(Position());
intx_t lo = ImmLo(offset);
intx_t hi = ImmHi(offset);
if (!IsUTypeImm(hi)) {
FATAL("Jump distance exceeds 2GB!");
}
EmitUType(hi, FAR_TMP, AUIPC);
EmitIType(lo, FAR_TMP, F3_0, ZR, JALR);
}
}
}
void MicroAssembler::EmitCBranch(Register rs1p, Label* label, COpcode op) {
intptr_t offset;
if (label->IsBound()) {
offset = label->Position() - Position();
} else {
offset = label->link_cb(Position());
}
if (!IsCBImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Emit16(op | EncodeCRs1p(rs1p) | EncodeCBImm(offset));
}
void MicroAssembler::EmitCJump(Label* label, COpcode op) {
intptr_t offset;
if (label->IsBound()) {
offset = label->Position() - Position();
} else {
offset = label->link_cj(Position());
}
if (!IsCJImm(offset)) {
FATAL("Incorrect Assembler::kNearJump");
}
Emit16(op | EncodeCJImm(offset));
}
void MicroAssembler::EmitRType(Funct5 funct5,
std::memory_order order,
Register rs2,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
intptr_t funct7 = funct5 << 2;
switch (order) {
case std::memory_order_acq_rel:
funct7 |= 0b11;
break;
case std::memory_order_acquire:
funct7 |= 0b10;
break;
case std::memory_order_release:
funct7 |= 0b01;
break;
case std::memory_order_relaxed:
funct7 |= 0b00;
break;
default:
FATAL("Invalid memory order");
}
EmitRType((Funct7)funct7, rs2, rs1, funct3, rd, opcode);
}
void MicroAssembler::EmitRType(Funct7 funct7,
Register rs2,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
Funct3 funct3,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
RoundingMode round,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
Register rs2,
Register rs1,
RoundingMode round,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
Register rs2,
Register rs1,
Funct3 funct3,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
FRegister rs2,
FRegister rs1,
RoundingMode round,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitRType(Funct7 funct7,
intptr_t shamt,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFunct7(funct7);
e |= EncodeShamt(shamt);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitR4Type(FRegister rs3,
Funct2 funct2,
FRegister rs2,
FRegister rs1,
RoundingMode round,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeFRs3(rs3);
e |= EncodeFunct2(funct2);
e |= EncodeFRs2(rs2);
e |= EncodeFRs1(rs1);
e |= EncodeRoundingMode(round);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitIType(intptr_t imm,
Register rs1,
Funct3 funct3,
Register rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeITypeImm(imm);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitIType(intptr_t imm,
Register rs1,
Funct3 funct3,
FRegister rd,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeITypeImm(imm);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeFRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitSType(intptr_t imm,
Register rs2,
Register rs1,
Funct3 funct3,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeSTypeImm(imm);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitSType(intptr_t imm,
FRegister rs2,
Register rs1,
Funct3 funct3,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeSTypeImm(imm);
e |= EncodeFRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitBType(intptr_t imm,
Register rs2,
Register rs1,
Funct3 funct3,
Opcode opcode) {
uint32_t e = 0;
e |= EncodeBTypeImm(imm);
e |= EncodeRs2(rs2);
e |= EncodeRs1(rs1);
e |= EncodeFunct3(funct3);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitUType(intptr_t imm, Register rd, Opcode opcode) {
uint32_t e = 0;
e |= EncodeUTypeImm(imm);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
void MicroAssembler::EmitJType(intptr_t imm, Register rd, Opcode opcode) {
uint32_t e = 0;
e |= EncodeJTypeImm(imm);
e |= EncodeRd(rd);
e |= EncodeOpcode(opcode);
Emit32(e);
}
Assembler::Assembler(ObjectPoolBuilder* object_pool_builder,
intptr_t far_branch_level)
: MicroAssembler(object_pool_builder, far_branch_level, RV_baseline),
constant_pool_allowed_(false) {
generate_invoke_write_barrier_wrapper_ = [&](Register reg) {
// Note this does not destroy RA.
lx(TMP,
Address(THR, target::Thread::write_barrier_wrappers_thread_offset(reg)));
jalr(TMP, TMP);
};
generate_invoke_array_write_barrier_ = [&]() {
Call(
Address(THR, target::Thread::array_write_barrier_entry_point_offset()));
};
}
void Assembler::PushRegister(Register r) {
ASSERT(r != SP);
subi(SP, SP, target::kWordSize);
sx(r, Address(SP, 0));
}
void Assembler::PopRegister(Register r) {
ASSERT(r != SP);
lx(r, Address(SP, 0));
addi(SP, SP, target::kWordSize);
}
void Assembler::PushRegisterPair(Register r0, Register r1) {
ASSERT(r0 != SP);
ASSERT(r1 != SP);
subi(SP, SP, 2 * target::kWordSize);
sx(r1, Address(SP, target::kWordSize));
sx(r0, Address(SP, 0));
}
void Assembler::PopRegisterPair(Register r0, Register r1) {
ASSERT(r0 != SP);
ASSERT(r1 != SP);
lx(r1, Address(SP, target::kWordSize));
lx(r0, Address(SP, 0));
addi(SP, SP, 2 * target::kWordSize);
}
void Assembler::PushRegisters(const RegisterSet& regs) {
// The order in which the registers are pushed must match the order
// in which the registers are encoded in the safepoint's stack map.
intptr_t size = regs.SpillSize();
if (size == 0) {
return; // Skip no-op SP update.
}
subi(SP, SP, size);
intptr_t offset = size;
for (intptr_t i = kNumberOfFpuRegisters - 1; i >= 0; i--) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
offset -= kFpuRegisterSize;
fsd(reg, Address(SP, offset));
}
}
for (intptr_t i = kNumberOfCpuRegisters - 1; i >= 0; i--) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
offset -= target::kWordSize;
sx(reg, Address(SP, offset));
}
}
ASSERT(offset == 0);
}
void Assembler::PopRegisters(const RegisterSet& regs) {
// The order in which the registers are pushed must match the order
// in which the registers are encoded in the safepoint's stack map.
intptr_t size = regs.SpillSize();
if (size == 0) {
return; // Skip no-op SP update.
}
intptr_t offset = 0;
for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
lx(reg, Address(SP, offset));
offset += target::kWordSize;
}
}
for (intptr_t i = 0; i < kNumberOfFpuRegisters; i++) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
fld(reg, Address(SP, offset));
offset += kFpuRegisterSize;
}
}
ASSERT(offset == size);
addi(SP, SP, size);
}
void Assembler::PushRegistersInOrder(std::initializer_list<Register> regs) {
intptr_t offset = regs.size() * target::kWordSize;
subi(SP, SP, offset);
for (Register reg : regs) {
ASSERT(reg != SP);
offset -= target::kWordSize;
sx(reg, Address(SP, offset));
}
}
void Assembler::PushNativeCalleeSavedRegisters() {
RegisterSet regs(kAbiPreservedCpuRegs, kAbiPreservedFpuRegs);
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
(regs.FpuRegisterCount() * sizeof(double));
subi(SP, SP, size);
intptr_t offset = 0;
for (intptr_t i = 0; i < kNumberOfFpuRegisters; i++) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
fsd(reg, Address(SP, offset));
offset += sizeof(double);
}
}
for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
sx(reg, Address(SP, offset));
offset += target::kWordSize;
}
}
ASSERT(offset == size);
}
void Assembler::PopNativeCalleeSavedRegisters() {
RegisterSet regs(kAbiPreservedCpuRegs, kAbiPreservedFpuRegs);
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
(regs.FpuRegisterCount() * sizeof(double));
intptr_t offset = 0;
for (intptr_t i = 0; i < kNumberOfFpuRegisters; i++) {
FRegister reg = static_cast<FRegister>(i);
if (regs.ContainsFpuRegister(reg)) {
fld(reg, Address(SP, offset));
offset += sizeof(double);
}
}
for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) {
Register reg = static_cast<Register>(i);
if (regs.ContainsRegister(reg)) {
lx(reg, Address(SP, offset));
offset += target::kWordSize;
}
}
ASSERT(offset == size);
addi(SP, SP, size);
}
void Assembler::ExtendValue(Register rd, Register rn, OperandSize sz) {
// For non-word bitsizes, use slli + srli for zero extension and slli + srai
// for sign extension when there is no single instruction to perform it.
switch (sz) {
#if XLEN == 64
case kEightBytes:
if (rd == rn) return; // No operation needed.
return mv(rd, rn);
case kUnsignedFourBytes:
if (Supports(RV_Zba)) {
return zextw(rd, rn);
}
slli(rd, rn, XLEN - 32);
return srli(rd, rn, XLEN - 32);
case kFourBytes:
return sextw(rd, rn);
#elif XLEN == 32
case kUnsignedFourBytes:
case kFourBytes:
if (rd == rn) return; // No operation needed.
return mv(rd, rn);
#else
#error Unexpected XLEN.
#endif
case kUnsignedTwoBytes:
if (Supports(RV_Zbb)) {
return zexth(rd, rn);
}
slli(rd, rn, XLEN - 16);
return srli(rd, rn, XLEN - 16);
case kTwoBytes:
if (Supports(RV_Zbb)) {
return sexth(rd, rn);
}
slli(rd, rn, XLEN - 16);
return srai(rd, rn, XLEN - 16);
case kUnsignedByte:
return andi(rd, rn, kMaxUint8);
case kByte:
if (Supports(RV_Zbb)) {
return sextb(rd, rn);
}
slli(rd, rn, XLEN - 8);
return srai(rd, rn, XLEN - 8);
default:
UNIMPLEMENTED();
break;
}
UNIMPLEMENTED();
}
void Assembler::ExtendAndSmiTagValue(Register rd, Register rn, OperandSize sz) {
if (sz == kWordBytes) {
SmiTag(rd, rn);
return;
}
switch (sz) {
#if XLEN == 64
case kUnsignedFourBytes:
slli(rd, rn, XLEN - kBitsPerInt32);
srli(rd, rd, XLEN - kBitsPerInt32 - kSmiTagShift);
return;
case kFourBytes:
slli(rd, rn, XLEN - kBitsPerInt32);
srai(rd, rd, XLEN - kBitsPerInt32 - kSmiTagShift);
return;
#endif
case kUnsignedTwoBytes:
slli(rd, rn, XLEN - kBitsPerInt16);
srli(rd, rd, XLEN - kBitsPerInt16 - kSmiTagShift);
return;
case kTwoBytes:
slli(rd, rn, XLEN - kBitsPerInt16);
srai(rd, rd, XLEN - kBitsPerInt16 - kSmiTagShift);
return;
case kUnsignedByte:
slli(rd, rn, XLEN - kBitsPerInt8);
srli(rd, rd, XLEN - kBitsPerInt8 - kSmiTagShift);
return;
case kByte:
slli(rd, rn, XLEN - kBitsPerInt8);
srai(rd, rd, XLEN - kBitsPerInt8 - kSmiTagShift);
return;
default:
UNIMPLEMENTED();
break;
}
}
// Unconditional jump to a given address in memory. Clobbers TMP.
void Assembler::Jump(const Address& address) {
lx(TMP2, address);
jr(TMP2);
}
void Assembler::TsanLoadAcquire(Register dst,
const Address& addr,
OperandSize size) {
ASSERT(addr.base() != SP);
ASSERT(addr.base() != FP);
ASSERT(dst != SP);
ASSERT(dst != FP);
Comment("TsanLoadAcquire");
RegisterSet registers(kDartVolatileCpuRegs & ~(1 << dst),
kAbiVolatileFpuRegs);
subi(SP, SP, 4 * target::kWordSize);
sx(RA, Address(SP, 3 * target::kWordSize));
sx(FP, Address(SP, 2 * target::kWordSize));
sx(CODE_REG, Address(SP, 1 * target::kWordSize));
sx(PP, Address(SP, 0 * target::kWordSize));
addi(FP, SP, 4 * target::kWordSize);
PushRegisters(registers);
ReserveAlignedFrameSpace(0);
AddImmediate(A0, addr.base(), addr.offset());
LoadImmediate(A1, static_cast<intx_t>(std::memory_order_acquire));
switch (size) {
case kEightBytes:
lx(TMP2, compiler::Address(
THR, kTsanAtomic64LoadRuntimeEntry.OffsetFromThread()));
break;
case kUnsignedFourBytes:
lx(TMP2, compiler::Address(
THR, kTsanAtomic32LoadRuntimeEntry.OffsetFromThread()));
break;
default:
UNIMPLEMENTED();
break;
}
sx(TMP2, compiler::Address(THR, target::Thread::vm_tag_offset()));
jalr(TMP2);
LoadImmediate(TMP2, VMTag::kDartTagId);
sx(TMP2, compiler::Address(THR, target::Thread::vm_tag_offset()));
MoveRegister(dst, A0);
subi(SP, FP, registers.SpillSize() + 4 * target::kWordSize);
PopRegisters(registers);
subi(SP, FP, 4 * target::kWordSize);
lx(PP, Address(SP, 0 * target::kWordSize));
lx(CODE_REG, Address(SP, 1 * target::kWordSize));
lx(FP, Address(SP, 2 * target::kWordSize));
lx(RA, Address(SP, 3 * target::kWordSize));
addi(SP, SP, 4 * target::kWordSize);
}
void Assembler::TsanStoreRelease(Register src,
const Address& addr,
OperandSize size) {
ASSERT(addr.base() != SP);
ASSERT(addr.base() != FP);
ASSERT(src != SP);
ASSERT(src != FP);
Comment("TsanStoreRelease");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
if (src == A0) {
MoveRegister(A1, src);
AddImmediate(A0, addr.base(), addr.offset());
} else {
AddImmediate(A0, addr.base(), addr.offset());
MoveRegister(A1, src);
}
LoadImmediate(A2, static_cast<intx_t>(std::memory_order_release));
switch (size) {
case kEightBytes:
rt.Call(kTsanAtomic64StoreRuntimeEntry, /*argument_count=*/3);
break;
case kFourBytes:
case kUnsignedFourBytes:
rt.Call(kTsanAtomic32StoreRuntimeEntry, /*argument_count=*/3);
break;
default:
UNIMPLEMENTED();
break;
}
}
void Assembler::TsanRead(Register addr, intptr_t size) {
Comment("TsanRead");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(A0, addr);
switch (size) {
case 1:
rt.Call(kTsanRead1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanRead2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanRead4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanRead8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanRead16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
void Assembler::TsanWrite(Register addr, intptr_t size) {
Comment("TsanWrite");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(A0, addr);
switch (size) {
case 1:
rt.Call(kTsanWrite1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanWrite2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanWrite4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanWrite8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanWrite16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
void Assembler::LoadAcquire(Register dst,
const Address& address,
OperandSize size) {
ASSERT(dst != address.base());
if (FLAG_target_thread_sanitizer) {
TsanLoadAcquire(dst, address, size);
return;
}
if (Supports(RV_Zalasr)) {
Address addr = PrepareAtomicOffset(address.base(), address.offset());
switch (size) {
#if XLEN == 64
case kEightBytes:
ld(dst, addr, std::memory_order_acquire);
break;
#endif
case kFourBytes:
lw(dst, addr, std::memory_order_acquire);
break;
case kTwoBytes:
lh(dst, addr, std::memory_order_acquire);
break;
case kByte:
lb(dst, addr, std::memory_order_acquire);
break;
default:
UNREACHABLE();
}
} else {
Load(dst, address, size);
fence(HartEffects::kRead, HartEffects::kMemory);
}
}
void Assembler::StoreRelease(Register src,
const Address& address,
OperandSize size) {
if (FLAG_target_thread_sanitizer) {
TsanStoreRelease(src, address, size);
return;
}
if (Supports(RV_Zalasr)) {
Address addr = PrepareAtomicOffset(address.base(), address.offset());
switch (size) {
#if XLEN == 64
case kEightBytes:
sd(src, addr, std::memory_order_release);
break;
#endif
case kUnsignedFourBytes:
case kFourBytes:
sw(src, addr, std::memory_order_release);
break;
case kUnsignedTwoBytes:
case kTwoBytes:
sh(src, addr, std::memory_order_release);
break;
case kUnsignedByte:
case kByte:
sb(src, addr, std::memory_order_release);
break;
default:
UNREACHABLE();
}
} else {
fence(HartEffects::kMemory, HartEffects::kWrite);
Store(src, address, size);
}
}
void Assembler::CompareWithMemoryValue(Register value,
Address address,
OperandSize size) {
#if XLEN >= 64
ASSERT(size == kEightBytes || size == kFourBytes);
if (size == kFourBytes) {
lw(TMP2, address);
} else {
ld(TMP2, address);
}
#else
ASSERT_EQUAL(size, kFourBytes);
lx(TMP2, address);
#endif
CompareRegisters(value, TMP2);
}
void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
if (frame_space != 0) {
addi(SP, SP, -frame_space);
}
const intptr_t kAbiStackAlignment = 16; // For both 32 and 64 bit.
andi(SP, SP, ~(kAbiStackAlignment - 1));
}
// In debug mode, this generates code to check that:
// FP + kExitLinkSlotFromEntryFp == SP
// or triggers breakpoint otherwise.
void Assembler::EmitEntryFrameVerification() {
#if defined(DEBUG)
Label done;
ASSERT(!constant_pool_allowed());
LoadImmediate(TMP, target::frame_layout.exit_link_slot_from_entry_fp *
target::kWordSize);
add(TMP, TMP, FPREG);
beq(TMP, SPREG, &done, kNearJump);
Breakpoint();
Bind(&done);
#endif
}
void Assembler::CompareRegisters(Register rn, Register rm) {
ASSERT(deferred_compare_ == kNone);
deferred_compare_ = kCompareReg;
deferred_left_ = rn;
deferred_reg_ = rm;
}
void Assembler::CompareObjectRegisters(Register rn, Register rm) {
CompareRegisters(rn, rm);
}
void Assembler::TestRegisters(Register rn, Register rm) {
ASSERT(deferred_compare_ == kNone);
deferred_compare_ = kTestReg;
deferred_left_ = rn;
deferred_reg_ = rm;
}
void Assembler::BranchIf(Condition condition,
Label* label,
JumpDistance distance) {
ASSERT(deferred_compare_ != kNone);
if (deferred_compare_ == kCompareImm || deferred_compare_ == kCompareReg) {
Register left = deferred_left_;
Register right;
if (deferred_compare_ == kCompareImm) {
if (deferred_imm_ == 0) {
right = ZR;
} else {
LoadImmediate(TMP2, deferred_imm_);
right = TMP2;
}
} else {
right = deferred_reg_;
}
switch (condition) {
case EQUAL:
beq(left, right, label, distance);
break;
case NOT_EQUAL:
bne(left, right, label, distance);
break;
case LESS:
blt(left, right, label, distance);
break;
case LESS_EQUAL:
ble(left, right, label, distance);
break;
case GREATER_EQUAL:
bge(left, right, label, distance);
break;
case GREATER:
bgt(left, right, label, distance);
break;
case UNSIGNED_LESS:
bltu(left, right, label, distance);
break;
case UNSIGNED_LESS_EQUAL:
bleu(left, right, label, distance);
break;
case UNSIGNED_GREATER_EQUAL:
bgeu(left, right, label, distance);
break;
case UNSIGNED_GREATER:
bgtu(left, right, label, distance);
break;
case OVERFLOW:
case NO_OVERFLOW:
FATAL("Use Add/Subtract/MultiplyBranchOverflow instead.");
default:
UNREACHABLE();
}
} else if (deferred_compare_ == kTestImm || deferred_compare_ == kTestReg) {
uintx_t uimm = deferred_imm_;
if (deferred_compare_ == kTestReg) {
and_(TMP2, deferred_left_, deferred_reg_);
} else if (IsITypeImm(deferred_imm_)) {
andi(TMP2, deferred_left_, deferred_imm_);
} else if (Utils::IsPowerOfTwo(uimm)) {
intptr_t shift = Utils::ShiftForPowerOfTwo(uimm);
Register sign;
if (shift == XLEN - 1) {
sign = deferred_left_;
} else {
slli(TMP2, deferred_left_, XLEN - 1 - shift);
sign = TMP2;
}
switch (condition) {
case ZERO:
bgez(sign, label, distance);
break;
case NOT_ZERO:
bltz(sign, label, distance);
break;
default:
UNREACHABLE();
}
deferred_compare_ = kNone; // Consumed.
return;
} else {
AndImmediate(TMP2, deferred_left_, deferred_imm_);
}
switch (condition) {
case ZERO:
beqz(TMP2, label, distance);
break;
case NOT_ZERO:
bnez(TMP2, label, distance);
break;
default:
UNREACHABLE();
}
} else {
UNREACHABLE();
}
deferred_compare_ = kNone; // Consumed.
}
void Assembler::SetIf(Condition condition, Register rd) {
ASSERT(deferred_compare_ != kNone);
if (deferred_compare_ == kCompareImm) {
if (deferred_imm_ == 0) {
deferred_compare_ = kCompareReg;
deferred_reg_ = ZR;
SetIf(condition, rd);
return;
}
if (!IsITypeImm(deferred_imm_) || !IsITypeImm(deferred_imm_ + 1)) {
LoadImmediate(TMP2, deferred_imm_);
deferred_compare_ = kCompareReg;
deferred_reg_ = TMP2;
SetIf(condition, rd);
return;
}
Register left = deferred_left_;
intx_t right = deferred_imm_;
switch (condition) {
case EQUAL:
subi(rd, left, right);
seqz(rd, rd);
break;
case NOT_EQUAL:
subi(rd, left, right);
snez(rd, rd);
break;
case LESS:
slti(rd, left, right);
break;
case LESS_EQUAL:
slti(rd, left, right + 1);
break;
case GREATER_EQUAL:
slti(rd, left, right);
xori(rd, rd, 1);
break;
case GREATER:
slti(rd, left, right + 1);
xori(rd, rd, 1);
break;
case UNSIGNED_LESS:
sltiu(rd, left, right);
break;
case UNSIGNED_LESS_EQUAL:
sltiu(rd, left, right + 1);
break;
case UNSIGNED_GREATER_EQUAL:
sltiu(rd, left, right);
xori(rd, rd, 1);
break;
case UNSIGNED_GREATER:
sltiu(rd, left, right + 1);
xori(rd, rd, 1);
break;
default:
UNREACHABLE();
}
} else if (deferred_compare_ == kCompareReg) {
Register left = deferred_left_;
Register right = deferred_reg_;
switch (condition) {
case EQUAL:
if (right == ZR) {
seqz(rd, left);
} else {
xor_(rd, left, right);
seqz(rd, rd);
}
break;
case NOT_EQUAL:
if (right == ZR) {
snez(rd, left);
} else {
xor_(rd, left, right);
snez(rd, rd);
}
break;
case LESS:
slt(rd, left, right);
break;
case LESS_EQUAL:
slt(rd, right, left);
xori(rd, rd, 1);
break;
case GREATER_EQUAL:
slt(rd, left, right);
xori(rd, rd, 1);
break;
case GREATER:
slt(rd, right, left);
break;
case UNSIGNED_LESS:
sltu(rd, left, right);
break;
case UNSIGNED_LESS_EQUAL:
sltu(rd, right, left);
xori(rd, rd, 1);
break;
case UNSIGNED_GREATER_EQUAL:
sltu(rd, left, right);
xori(rd, rd, 1);
break;
case UNSIGNED_GREATER:
sltu(rd, right, left);
break;
default:
UNREACHABLE();
}
} else if (deferred_compare_ == kTestImm) {
uintx_t uimm = deferred_imm_;
if (deferred_imm_ == 1) {
switch (condition) {
case ZERO:
andi(rd, deferred_left_, 1);
xori(rd, rd, 1);
break;
case NOT_ZERO:
andi(rd, deferred_left_, 1);
break;
default:
UNREACHABLE();
}
} else if (Supports(RV_Zbs) && Utils::IsPowerOfTwo(uimm)) {
switch (condition) {
case ZERO:
bexti(rd, deferred_left_, Utils::ShiftForPowerOfTwo(uimm));
xori(rd, rd, 1);
break;
case NOT_ZERO:
bexti(rd, deferred_left_, Utils::ShiftForPowerOfTwo(uimm));
break;
default:
UNREACHABLE();
}
} else {
AndImmediate(rd, deferred_left_, deferred_imm_);
switch (condition) {
case ZERO:
seqz(rd, rd);
break;
case NOT_ZERO:
snez(rd, rd);
break;
default:
UNREACHABLE();
}
}
} else if (deferred_compare_ == kTestReg) {
and_(rd, deferred_left_, deferred_reg_);
switch (condition) {
case ZERO:
seqz(rd, rd);
break;
case NOT_ZERO:
snez(rd, rd);
break;
default:
UNREACHABLE();
}
} else {
UNREACHABLE();
}
deferred_compare_ = kNone; // Consumed.
}
void Assembler::BranchIfZero(Register rn, Label* label, JumpDistance distance) {
beqz(rn, label, distance);
}
void Assembler::BranchIfBit(Register rn,
intptr_t bit_number,
Condition condition,
Label* label,
JumpDistance distance) {
ASSERT(rn != TMP2);
andi(TMP2, rn, 1 << bit_number);
if (condition == ZERO) {
beqz(TMP2, label, distance);
} else if (condition == NOT_ZERO) {
bnez(TMP2, label, distance);
} else {
UNREACHABLE();
}
}
void Assembler::BranchIfNotSmi(Register reg,
Label* label,
JumpDistance distance) {
ASSERT(reg != TMP2);
andi(TMP2, reg, kSmiTagMask);
bnez(TMP2, label, distance);
}
void Assembler::BranchIfSmi(Register reg, Label* label, JumpDistance distance) {
ASSERT(reg != TMP2);
andi(TMP2, reg, kSmiTagMask);
beqz(TMP2, label, distance);
}
void Assembler::ArithmeticShiftRightImmediate(Register dst,
Register src,
int32_t shift,
OperandSize sz) {
ASSERT(sz == kEightBytes || sz == kFourBytes);
ASSERT((shift >= 0) && (shift < OperandSizeInBits(sz)));
if (shift == 0) {
return ExtendValue(dst, src, sz);
}
#if XLEN >= 64
if (sz == kFourBytes) {
return sraiw(dst, src, shift);
}
#endif
srai(dst, src, shift);
}
void Assembler::CompareWords(Register reg1,
Register reg2,
intptr_t offset,
Register count,
Register temp,
Label* equals) {
Label loop;
Bind(&loop);
BranchIfZero(count, equals, Assembler::kNearJump);
AddImmediate(count, -1);
lx(temp, FieldAddress(reg1, offset));
lx(TMP, FieldAddress(reg2, offset));
addi(reg1, reg1, target::kWordSize);
addi(reg2, reg2, target::kWordSize);
beq(temp, TMP, &loop, Assembler::kNearJump);
}
void Assembler::JumpAndLink(intptr_t target_code_pool_index,
CodeEntryKind entry_kind) {
// Avoid clobbering CODE_REG when invoking code in precompiled mode.
// We don't actually use CODE_REG in the callee and caller might
// be using CODE_REG for a live value (e.g. a value that is alive
// across invocation of a shared stub like the one we use for
// allocating Mint boxes).
const Register code_reg = FLAG_precompiled_mode ? TMP : CODE_REG;
LoadWordFromPoolIndex(code_reg, target_code_pool_index);
Call(FieldAddress(code_reg, target::Code::entry_point_offset(entry_kind)));
}
void Assembler::JumpAndLink(
const Code& target,
ObjectPoolBuilderEntry::Patchability patchable,
CodeEntryKind entry_kind,
ObjectPoolBuilderEntry::SnapshotBehavior snapshot_behavior) {
const intptr_t index = object_pool_builder().FindObject(
ToObject(target), patchable, snapshot_behavior);
JumpAndLink(index, entry_kind);
}
void Assembler::JumpAndLinkWithEquivalence(const Code& target,
const Object& equivalence,
CodeEntryKind entry_kind) {
const intptr_t index =
object_pool_builder().FindObject(ToObject(target), equivalence);
JumpAndLink(index, entry_kind);
}
void Assembler::Call(Address target) {
lx(RA, target);
jalr(RA);
}
void Assembler::Call(Register target) {
jalr(target);
}
void Assembler::AddShifted(Register dest,
Register base,
Register index,
intx_t shift) {
if (shift == 0) {
add(dest, index, base);
} else if (Supports(RV_Zba) && (shift == 1)) {
sh1add(dest, index, base);
} else if (Supports(RV_Zba) && (shift == 2)) {
sh2add(dest, index, base);
} else if (Supports(RV_Zba) && (shift == 3)) {
sh3add(dest, index, base);
} else if (shift < 0) {
if (base != dest) {
srai(dest, index, -shift);
add(dest, dest, base);
} else {
srai(TMP2, index, -shift);
add(dest, TMP2, base);
}
} else {
if (base != dest) {
slli(dest, index, shift);
add(dest, dest, base);
} else {
slli(TMP2, index, shift);
add(dest, TMP2, base);
}
}
}
void Assembler::AddImmediate(Register rd,
Register rs1,
intx_t imm,
OperandSize sz) {
if ((imm == 0) && (rd == rs1)) {
return;
}
if (IsITypeImm(imm)) {
addi(rd, rs1, imm);
} else {
ASSERT(rs1 != TMP2);
LoadImmediate(TMP2, imm);
add(rd, rs1, TMP2);
}
}
void Assembler::MulImmediate(Register rd,
Register rs1,
intx_t imm,
OperandSize sz) {
if (Utils::IsPowerOfTwo(imm)) {
const intx_t shift = Utils::ShiftForPowerOfTwo(imm);
#if XLEN >= 64
ASSERT(sz == kFourBytes || sz == kEightBytes);
if (sz == kFourBytes) {
slliw(rd, rs1, shift);
} else {
slli(rd, rs1, shift);
}
#else
ASSERT(sz == kFourBytes);
slli(rd, rs1, shift);
#endif
} else {
LoadImmediate(TMP, imm);
#if XLEN >= 64
ASSERT(sz == kFourBytes || sz == kEightBytes);
if (sz == kFourBytes) {
mulw(rd, rs1, TMP);
} else {
mul(rd, rs1, TMP);
}
#else
ASSERT(sz == kFourBytes);
mul(rd, rs1, TMP);
#endif
}
}
void Assembler::AndImmediate(Register rd,
Register rs1,
intx_t imm,
OperandSize sz) {
ASSERT(sz == kEightBytes || sz == kUnsignedFourBytes || sz == kFourBytes);
const intptr_t bit_size = OperandSizeInBits(sz);
ASSERT(Utils::IsInt(bit_size, imm) || Utils::IsUint(bit_size, imm));
// Clear the upper bits of the immediate for non-word-sized uses.
uintx_t uimm = bit_size == XLEN ? imm : static_cast<uint32_t>(imm);
if (bit_size == XLEN && imm == -1) {
// All bits set, so allow this operation to be a no-op if rd == rs1.
MoveRegister(rd, rs1);
} else if (bit_size < XLEN && static_cast<int32_t>(imm) == -1) {
ExtendValue(rd, rs1, kUnsignedFourBytes);
} else if (IsITypeImm(uimm)) {
andi(rd, rs1, uimm);
} else if (Supports(RV_Zbs) && Utils::IsPowerOfTwo(~uimm)) {
bclri(rd, rs1, Utils::ShiftForPowerOfTwo(~uimm));
} else if (Utils::IsPowerOfTwo(uimm + 1)) {
intptr_t shift = Utils::ShiftForPowerOfTwo(uimm + 1);
if (Supports(RV_Zbb) && (shift == 16)) {
zexth(rd, rs1);
} else {
slli(rd, rs1, XLEN - shift);
srli(rd, rd, XLEN - shift);
}
} else {
ASSERT(rs1 != TMP2);
LoadImmediate(TMP2, uimm);
and_(rd, rs1, TMP2);
}
}
void Assembler::OrImmediate(Register rd,
Register rs1,
intx_t imm,
OperandSize sz) {
uintx_t uimm = imm;
if (imm == 0) {
MoveRegister(rd, rs1);
} else if (IsITypeImm(imm)) {
ori(rd, rs1, imm);
} else if (Supports(RV_Zbs) && Utils::IsPowerOfTwo(uimm)) {
bseti(rd, rs1, Utils::ShiftForPowerOfTwo(uimm));
} else {
ASSERT(rs1 != TMP2);
LoadImmediate(TMP2, imm);
or_(rd, rs1, TMP2);
}
}
void Assembler::XorImmediate(Register rd,
Register rs1,
intx_t imm,
OperandSize sz) {
uintx_t uimm = imm;
if (imm == 0) {
MoveRegister(rd, rs1);
} else if (IsITypeImm(imm)) {
xori(rd, rs1, imm);
} else if (Supports(RV_Zbs) && Utils::IsPowerOfTwo(uimm)) {
binvi(rd, rs1, Utils::ShiftForPowerOfTwo(uimm));
} else {
ASSERT(rs1 != TMP2);
LoadImmediate(TMP2, imm);
xor_(rd, rs1, TMP2);
}
}
void Assembler::LslImmediate(Register rd,
Register rn,
int32_t shift,
OperandSize sz) {
ASSERT(sz == kEightBytes || sz == kFourBytes || sz == kUnsignedFourBytes);
ASSERT((shift >= 0) && (shift < OperandSizeInBits(sz)));
if (shift == 0) {
return ExtendValue(rd, rn, sz);
}
#if XLEN >= 64
if (sz == kFourBytes) {
return slliw(rd, rn, shift);
}
if (sz == kUnsignedFourBytes) {
// Not slliuw even when available. That zero extends the input, not the
// output.
// Clear upper bits in addition to the shift.
slli(rd, rn, shift + (XLEN / 2));
return srli(rd, rn, XLEN / 2);
}
#endif
slli(rd, rn, shift);
}
void Assembler::TestImmediate(Register rn, intx_t imm, OperandSize sz) {
ASSERT(deferred_compare_ == kNone);
deferred_compare_ = kTestImm;
deferred_left_ = rn;
deferred_imm_ = imm;
}
void Assembler::CompareImmediate(Register rn, intx_t imm, OperandSize sz) {
ASSERT(deferred_compare_ == kNone);
deferred_compare_ = kCompareImm;
deferred_left_ = rn;
deferred_imm_ = imm;
}
Address Assembler::PrepareLargeOffset(Register base, int32_t offset) {
ASSERT(base != TMP2);
if (IsITypeImm(offset)) {
return Address(base, offset);
}
intx_t lo = ImmLo(offset);
intx_t hi = ImmHi(offset);
ASSERT(hi != 0);
lui(TMP2, hi);
add(TMP2, TMP2, base);
return Address(TMP2, lo);
}
Address Assembler::PrepareAtomicOffset(Register base, int32_t offset) {
ASSERT(base != TMP2);
if (offset == 0) {
return Address(base, 0);
}
AddImmediate(TMP2, base, offset);
return Address(TMP2, 0);
}
void Assembler::Load(Register dest, const Address& address, OperandSize sz) {
Address addr = PrepareLargeOffset(address.base(), address.offset());
switch (sz) {
#if XLEN == 64
case kEightBytes:
return ld(dest, addr);
case kUnsignedFourBytes:
return lwu(dest, addr);
#elif XLEN == 32
case kUnsignedFourBytes:
return lw(dest, addr);
#endif
case kFourBytes:
return lw(dest, addr);
case kUnsignedTwoBytes:
return lhu(dest, addr);
case kTwoBytes:
return lh(dest, addr);
case kUnsignedByte:
return lbu(dest, addr);
case kByte:
return lb(dest, addr);
default:
UNREACHABLE();
}
}
// For loading indexed payloads out of tagged objects like Arrays. If the
// payload objects are word-sized, use TIMES_HALF_WORD_SIZE if the contents of
// [index] is a Smi, otherwise TIMES_WORD_SIZE if unboxed.
void Assembler::LoadIndexedPayload(Register dest,
Register base,
int32_t payload_offset,
Register index,
ScaleFactor scale,
OperandSize sz) {
AddShifted(TMP, base, index, scale);
LoadFromOffset(dest, TMP, payload_offset - kHeapObjectTag, sz);
}
void Assembler::LoadSFromOffset(FRegister dest, Register base, int32_t offset) {
flw(dest, PrepareLargeOffset(base, offset));
}
void Assembler::LoadDFromOffset(FRegister dest, Register base, int32_t offset) {
fld(dest, PrepareLargeOffset(base, offset));
}
void Assembler::LoadFromStack(Register dst, intptr_t depth) {
LoadFromOffset(dst, SPREG, target::kWordSize * depth);
}
void Assembler::StoreToStack(Register src, intptr_t depth) {
StoreToOffset(src, SPREG, target::kWordSize * depth);
}
void Assembler::CompareToStack(Register src, intptr_t depth) {
CompareWithMemoryValue(src, Address(SPREG, target::kWordSize * depth));
}
void Assembler::Store(Register src, const Address& address, OperandSize sz) {
Address addr = PrepareLargeOffset(address.base(), address.offset());
switch (sz) {
#if XLEN == 64
case kEightBytes:
return sd(src, addr);
#endif
case kUnsignedFourBytes:
case kFourBytes:
return sw(src, addr);
case kUnsignedTwoBytes:
case kTwoBytes:
return sh(src, addr);
case kUnsignedByte:
case kByte:
return sb(src, addr);
default:
UNREACHABLE();
}
}
void Assembler::StoreSToOffset(FRegister src, Register base, int32_t offset) {
fsw(src, PrepareLargeOffset(base, offset));
}
void Assembler::StoreDToOffset(FRegister src, Register base, int32_t offset) {
fsd(src, PrepareLargeOffset(base, offset));
}
void Assembler::StoreBarrier(Register object,
Register value,
CanBeSmi can_value_be_smi,
Register scratch) {
// x.slot = x. Barrier should have be removed at the IL level.
ASSERT(object != value);
ASSERT(object != scratch);
ASSERT(value != scratch);
ASSERT(object != RA);
ASSERT(value != RA);
ASSERT(scratch != RA);
ASSERT(object != TMP2);
ASSERT(value != TMP2);
ASSERT(scratch != TMP2);
ASSERT(scratch != kNoRegister);
// In parallel, test whether
// - object is old and not remembered and value is new, or
// - object is old and value is old and not marked and concurrent marking is
// in progress
// If so, call the WriteBarrier stub, which will either add object to the
// store buffer (case 1) or add value to the marking stack (case 2).
// See RestorePinnedRegisters for why this can be `ble`.
// Compare UntaggedObject::StorePointer.
Label done;
if (can_value_be_smi == kValueCanBeSmi) {
BranchIfSmi(value, &done, kNearJump);
} else {
#if defined(DEBUG)
Label passed_check;
BranchIfNotSmi(value, &passed_check, kNearJump);
Breakpoint();
Bind(&passed_check);
#endif
}
lbu(scratch, FieldAddress(object, target::Object::tags_offset()));
lbu(TMP2, FieldAddress(value, target::Object::tags_offset()));
srli(scratch, scratch, target::UntaggedObject::kBarrierOverlapShift);
and_(scratch, scratch, TMP2);
ble(scratch, WRITE_BARRIER_STATE, &done, kNearJump);
Register objectForCall = object;
if (value != kWriteBarrierValueReg) {
// Unlikely. Only non-graph intrinsics.
// TODO(rmacnak): Shuffle registers in intrinsics.
if (object != kWriteBarrierValueReg) {
PushRegister(kWriteBarrierValueReg);
} else {
COMPILE_ASSERT(S3 != kWriteBarrierValueReg);
COMPILE_ASSERT(S4 != kWriteBarrierValueReg);
objectForCall = (value == S3) ? S4 : S3;
PushRegisterPair(kWriteBarrierValueReg, objectForCall);
mv(objectForCall, object);
}
mv(kWriteBarrierValueReg, value);
}
// Note this uses TMP as the link register, so RA remains preserved.
generate_invoke_write_barrier_wrapper_(objectForCall);
if (value != kWriteBarrierValueReg) {
if (object != kWriteBarrierValueReg) {
PopRegister(kWriteBarrierValueReg);
} else {
PopRegisterPair(kWriteBarrierValueReg, objectForCall);
}
}
Bind(&done);
}
void Assembler::ArrayStoreBarrier(Register object,
Register slot,
Register value,
CanBeSmi can_value_be_smi,
Register scratch) {
// TODO(riscv): Use RA2 to avoid spilling RA inline?
const bool spill_lr = true;
ASSERT(object != slot);
ASSERT(object != value);
ASSERT(object != scratch);
ASSERT(slot != value);
ASSERT(slot != scratch);
ASSERT(value != scratch);
ASSERT(object != RA);
ASSERT(slot != RA);
ASSERT(value != RA);
ASSERT(scratch != RA);
ASSERT(object != TMP2);
ASSERT(slot != TMP2);
ASSERT(value != TMP2);
ASSERT(scratch != TMP2);
ASSERT(scratch != kNoRegister);
// In parallel, test whether
// - object is old and not remembered and value is new, or
// - object is old and value is old and not marked and concurrent marking is
// in progress
// If so, call the WriteBarrier stub, which will either add object to the
// store buffer (case 1) or add value to the marking stack (case 2).
// See RestorePinnedRegisters for why this can be `ble`.
// Compare UntaggedObject::StorePointer.
Label done;
if (can_value_be_smi == kValueCanBeSmi) {
BranchIfSmi(value, &done, kNearJump);
} else {
#if defined(DEBUG)
Label passed_check;
BranchIfNotSmi(value, &passed_check, kNearJump);
Breakpoint();
Bind(&passed_check);
#endif
}
lbu(scratch, FieldAddress(object, target::Object::tags_offset()));
lbu(TMP2, FieldAddress(value, target::Object::tags_offset()));
srli(scratch, scratch, target::UntaggedObject::kBarrierOverlapShift);
and_(scratch, scratch, TMP2);
ble(scratch, WRITE_BARRIER_STATE, &done, kNearJump);
if (spill_lr) {
PushRegister(RA);
}
if ((object != kWriteBarrierObjectReg) || (value != kWriteBarrierValueReg) ||
(slot != kWriteBarrierSlotReg)) {
// Spill and shuffle unimplemented. Currently StoreIntoArray is only used
// from StoreIndexInstr, which gets these exact registers from the register
// allocator.
UNIMPLEMENTED();
}
generate_invoke_array_write_barrier_();
if (spill_lr) {
PopRegister(RA);
}
Bind(&done);
}
void Assembler::VerifyStoreNeedsNoWriteBarrier(Register object,
Register value) {
if (value == ZR) return;
// We can't assert the incremental barrier is not needed here, only the
// generational barrier. We sometimes omit the write barrier when 'value' is
// a constant, but we don't eagerly mark 'value' and instead assume it is also
// reachable via a constant pool, so it doesn't matter if it is not traced via
// 'object'.
Label done;
BranchIfSmi(value, &done, kNearJump);
lbu(TMP2, FieldAddress(value, target::Object::tags_offset()));
andi(TMP2, TMP2, 1 << target::UntaggedObject::kNewOrEvacuationCandidateBit);
beqz(TMP2, &done, kNearJump);
lbu(TMP2, FieldAddress(object, target::Object::tags_offset()));
andi(TMP2, TMP2, 1 << target::UntaggedObject::kOldAndNotRememberedBit);
beqz(TMP2, &done, kNearJump);
Stop("Write barrier is required");
Bind(&done);
}
void Assembler::StoreObjectIntoObjectNoBarrier(Register object,
const Address& dest,
const Object& value,
MemoryOrder memory_order,
OperandSize size) {
ASSERT(IsOriginalObject(value));
DEBUG_ASSERT(IsNotTemporaryScopedHandle(value));
// No store buffer update.
Register value_reg;
if (IsSameObject(compiler::NullObject(), value)) {
value_reg = NULL_REG;
} else if (target::IsSmi(value) && (target::ToRawSmi(value) == 0)) {
value_reg = ZR;
} else {
ASSERT(object != TMP);
LoadObject(TMP, value);
value_reg = TMP;
}
if (memory_order == kRelease) {
fence(HartEffects::kMemory, HartEffects::kWrite);
}
Store(value_reg, dest, size);
}
// Stores a non-tagged value into a heap object.
void Assembler::StoreInternalPointer(Register object,
const Address& dest,
Register value) {
sx(value, dest);
}
// Object pool, loading from pool, etc.
void Assembler::LoadPoolPointer(Register pp) {
CheckCodePointer();
lx(pp, FieldAddress(CODE_REG, target::Code::object_pool_offset()));
// When in the PP register, the pool pointer is untagged. When we
// push it on the stack with TagAndPushPP it is tagged again. PopAndUntagPP
// then untags when restoring from the stack. This will make loading from the
// object pool only one instruction for the first 4096 entries. Otherwise,
// because the offset wouldn't be aligned, it would be only one instruction
// for the first 64 entries.
subi(pp, pp, kHeapObjectTag);
set_constant_pool_allowed(pp == PP);
}
bool Assembler::CanLoadFromObjectPool(const Object& object) const {
ASSERT(IsOriginalObject(object));
if (!constant_pool_allowed()) {
return false;
}
DEBUG_ASSERT(IsNotTemporaryScopedHandle(object));
ASSERT(IsInOldSpace(object));
return true;
}
void Assembler::LoadNativeEntry(
Register dst,
const ExternalLabel* label,
ObjectPoolBuilderEntry::Patchability patchable) {
const intptr_t index =
object_pool_builder().FindNativeFunction(label, patchable);
LoadWordFromPoolIndex(dst, index);
}
void Assembler::LoadIsolate(Register dst) {
lx(dst, Address(THR, target::Thread::isolate_offset()));
}
void Assembler::LoadIsolateGroup(Register dst) {
lx(dst, Address(THR, target::Thread::isolate_group_offset()));
}
void Assembler::LoadImmediate(Register reg, intx_t imm) {
#if XLEN > 32
if (!Utils::IsInt(32, imm)) {
int shift = Utils::CountTrailingZeros64(imm);
if (IsITypeImm(imm >> shift)) {
li(reg, imm >> shift);
slli(reg, reg, shift);
return;
}
if ((shift >= 12) && IsUTypeImm(imm >> (shift - 12))) {
lui(reg, imm >> (shift - 12));
slli(reg, reg, shift - 12);
return;
}
if (constant_pool_allowed()) {
intptr_t index = object_pool_builder().FindImmediate(imm);
LoadWordFromPoolIndex(reg, index);
return;
}
intx_t lo = ImmLo(imm);
intx_t hi = imm - lo;
shift = Utils::CountTrailingZeros64(hi);
ASSERT(shift != 0);
LoadImmediate(reg, hi >> shift);
slli(reg, reg, shift);
if (lo != 0) {
addi(reg, reg, lo);
}
return;
}
#endif
intx_t lo = ImmLo(imm);
intx_t hi = ImmHi(imm);
if (hi == 0) {
addi(reg, ZR, lo);
} else {
lui(reg, hi);
if (lo != 0) {
#if XLEN == 32
addi(reg, reg, lo);
#else
addiw(reg, reg, lo);
#endif
}
}
}
void Assembler::LoadSImmediate(FRegister reg, float imms) {
uint32_t imm = bit_cast<uint32_t, float>(imms);
if (imm == 0) {
fmvwx(reg, ZR); // bit_cast uint32_t -> float
} else {
if (Supports(RV_Zfa)) {
for (intptr_t i = 0; i < 32; i++) {
if (kFlisConstants[i] == imm) {
flis(reg, i);
return;
}
}
}
ASSERT(constant_pool_allowed());
intptr_t index = object_pool_builder().FindImmediate(imm);
intptr_t offset = target::ObjectPool::element_offset(index);
LoadSFromOffset(reg, PP, offset);
}
}
void Assembler::LoadDImmediate(FRegister reg, double immd) {
uint64_t imm = bit_cast<uint64_t, double>(immd);
if (imm == 0) {
#if XLEN >= 64
fmvdx(reg, ZR); // bit_cast uint64_t -> double
#else
fcvtdwu(reg, ZR); // static_cast uint32_t -> double
#endif
} else {
if (Supports(RV_Zfa)) {
for (intptr_t i = 0; i < 32; i++) {
if (kFlidConstants[i] == imm) {
flid(reg, i);
return;
}
}
}
ASSERT(constant_pool_allowed());
intptr_t index = object_pool_builder().FindImmediate64(imm);
intptr_t offset = target::ObjectPool::element_offset(index);
LoadDFromOffset(reg, PP, offset);
}
}
void Assembler::LoadQImmediate(FRegister reg, simd128_value_t immq) {
UNREACHABLE(); // F registers cannot represent SIMD128.
}
// Load word from pool from the given offset using encoding that
// InstructionPattern::DecodeLoadWordFromPool can decode.
//
// Note: the function never clobbers TMP, TMP2 scratch registers.
void Assembler::LoadWordFromPoolIndex(Register dst,
intptr_t index,
Register pp) {
ASSERT((pp != PP) || constant_pool_allowed());
ASSERT(dst != pp);
const uint32_t offset = target::ObjectPool::element_offset(index);
// PP is untagged.
intx_t lo = ImmLo(offset);
intx_t hi = ImmHi(offset);
if (hi == 0) {
lx(dst, Address(pp, lo));
} else {
lui(dst, hi);
add(dst, dst, pp);
lx(dst, Address(dst, lo));
}
}
void Assembler::StoreWordToPoolIndex(Register src,
intptr_t index,
Register pp) {
ASSERT((pp != PP) || constant_pool_allowed());
ASSERT(src != pp);
const uint32_t offset = target::ObjectPool::element_offset(index);
// PP is untagged.
intx_t lo = ImmLo(offset);
intx_t hi = ImmHi(offset);
if (hi == 0) {
sx(src, Address(pp, lo));
} else {
lui(TMP, hi);
add(TMP, TMP, pp);
sx(src, Address(TMP, lo));
}
}
void Assembler::CompareObject(Register reg, const Object& object) {
ASSERT(IsOriginalObject(object));
if (IsSameObject(compiler::NullObject(), object)) {
CompareObjectRegisters(reg, NULL_REG);
} else if (target::IsSmi(object)) {
CompareImmediate(reg, target::ToRawSmi(object), kObjectBytes);
} else {
LoadObject(TMP, object);
CompareObjectRegisters(reg, TMP);
}
}
void Assembler::ExtractClassIdFromTags(Register result, Register tags) {
ASSERT(target::UntaggedObject::kClassIdTagPos == 12);
ASSERT(target::UntaggedObject::kClassIdTagSize == 20);
#if XLEN == 64
srliw(result, tags, target::UntaggedObject::kClassIdTagPos);
#else
srli(result, tags, target::UntaggedObject::kClassIdTagPos);
#endif
}
void Assembler::ExtractInstanceSizeFromTags(Register result, Register tags) {
ASSERT(target::UntaggedObject::kSizeTagPos == 8);
ASSERT(target::UntaggedObject::kSizeTagSize == 4);
srli(result, tags, target::UntaggedObject::kSizeTagPos);
andi(result, result, (1 << target::UntaggedObject::kSizeTagSize) - 1);
slli(result, result, target::ObjectAlignment::kObjectAlignmentLog2);
}
void Assembler::LoadClassId(Register result, Register object) {
ASSERT(target::UntaggedObject::kClassIdTagPos == 12);
ASSERT(target::UntaggedObject::kClassIdTagSize == 20);
#if XLEN == 64
lwu(result, FieldAddress(object, target::Object::tags_offset()));
#else
lw(result, FieldAddress(object, target::Object::tags_offset()));
#endif
srli(result, result, target::UntaggedObject::kClassIdTagPos);
}
void Assembler::LoadClassById(Register result, Register class_id) {
ASSERT(result != class_id);
const intptr_t table_offset =
target::IsolateGroup::cached_class_table_table_offset();
LoadIsolateGroup(result);
LoadFromOffset(result, result, table_offset);
AddShifted(result, result, class_id, target::kWordSizeLog2);
lx(result, Address(result, 0));
}
void Assembler::CompareClassId(Register object,
intptr_t class_id,
Register scratch) {
ASSERT(scratch != kNoRegister);
LoadClassId(scratch, object);
CompareImmediate(scratch, class_id);
}
// Note: input and output registers must be different.
void Assembler::LoadClassIdMayBeSmi(Register result, Register object) {
ASSERT(result != object);
ASSERT(result != TMP2);
ASSERT(object != TMP2);
li(result, kSmiCid);
Label done;
BranchIfSmi(object, &done, kNearJump);
LoadClassId(result, object);
Bind(&done);
}
void Assembler::LoadTaggedClassIdMayBeSmi(Register result, Register object) {
LoadClassIdMayBeSmi(result, object);
SmiTag(result);
}
void Assembler::EnsureHasClassIdInDEBUG(intptr_t cid,
Register src,
Register scratch,
bool can_be_null) {
#if defined(DEBUG)
Comment("Check that object in register has cid %" Pd "", cid);
Label matches;
LoadClassIdMayBeSmi(scratch, src);
CompareImmediate(scratch, cid);
BranchIf(EQUAL, &matches, Assembler::kNearJump);
if (can_be_null) {
CompareImmediate(scratch, kNullCid);
BranchIf(EQUAL, &matches, Assembler::kNearJump);
}
trap();
Bind(&matches);
#endif
}
void Assembler::EnterFrame(intptr_t frame_size) {
// N.B. The ordering here is important. We must never write beyond SP or
// it can be clobbered by a signal handler.
subi(SP, SP, frame_size + 2 * target::kWordSize);
sx(RA, Address(SP, frame_size + 1 * target::kWordSize));
sx(FP, Address(SP, frame_size + 0 * target::kWordSize));
addi(FP, SP, frame_size + 2 * target::kWordSize);
}
void Assembler::LeaveFrame() {
// N.B. The ordering here is important. We must never read beyond SP or
// it may have already been clobbered by a signal handler.
subi(SP, FP, 2 * target::kWordSize);
lx(FP, Address(SP, 0 * target::kWordSize));
lx(RA, Address(SP, 1 * target::kWordSize));
addi(SP, SP, 2 * target::kWordSize);
}
void Assembler::TransitionGeneratedToNative(Register destination,
Register new_exit_frame,
Register new_exit_through_ffi,
bool enter_safepoint) {
// Save exit frame information to enable stack walking.
sx(new_exit_frame,
Address(THR, target::Thread::top_exit_frame_info_offset()));
sx(new_exit_through_ffi,
Address(THR, target::Thread::exit_through_ffi_offset()));
Register tmp = new_exit_through_ffi;
#if defined(DEBUG)
ASSERT(T2 != TMP2);
mv(T2, TMP2); // BranchIf in VerifyInGenerated clobbers TMP2.
VerifyInGenerated(tmp);
mv(TMP2, T2);
#endif
// Mark that the thread is executing native code.
sx(destination, Address(THR, target::Thread::vm_tag_offset()));
li(tmp, target::Thread::native_execution_state());
sx(tmp, Address(THR, target::Thread::execution_state_offset()));
if (enter_safepoint) {
EnterFullSafepoint(tmp);
}
}
void Assembler::TransitionNativeToGenerated(Register state,
bool exit_safepoint,
bool set_tag) {
if (exit_safepoint) {
ExitFullSafepoint(state);
} else {
#if defined(DEBUG)
// Ensure we've already left the safepoint.
ASSERT(target::Thread::native_safepoint_state_acquired() != 0);
li(state, target::Thread::native_safepoint_state_acquired());
lx(RA, Address(THR, target::Thread::safepoint_state_offset()));
and_(RA, RA, state);
Label ok;
beqz(RA, &ok, Assembler::kNearJump);
Breakpoint();
Bind(&ok);
#endif
}
VerifyNotInGenerated(state);
// Mark that the thread is executing Dart code.
if (set_tag) {
li(state, target::Thread::vm_tag_dart_id());
sx(state, Address(THR, target::Thread::vm_tag_offset()));
}
li(state, target::Thread::generated_execution_state());
sx(state, Address(THR, target::Thread::execution_state_offset()));
// Reset exit frame information in Isolate's mutator thread structure.
sx(ZR, Address(THR, target::Thread::top_exit_frame_info_offset()));
sx(ZR, Address(THR, target::Thread::exit_through_ffi_offset()));
}
void Assembler::VerifyInGenerated(Register scratch) {
#if defined(DEBUG)
// Verify the thread is in generated.
Comment("VerifyInGenerated");
lx(scratch, Address(THR, target::Thread::execution_state_offset()));
Label ok;
CompareImmediate(scratch, target::Thread::generated_execution_state());
BranchIf(EQUAL, &ok, Assembler::kNearJump);
Breakpoint();
Bind(&ok);
#endif
}
void Assembler::VerifyNotInGenerated(Register scratch) {
#if defined(DEBUG)
// Verify the thread is in native or VM.
Comment("VerifyNotInGenerated");
lx(scratch, Address(THR, target::Thread::execution_state_offset()));
CompareImmediate(scratch, target::Thread::generated_execution_state());
Label ok;
BranchIf(NOT_EQUAL, &ok, Assembler::kNearJump);
Breakpoint();
Bind(&ok);
#endif
}
void Assembler::EnterFullSafepoint(Register state) {
// We generate the same number of instructions whether or not the slow-path is
// forced. This simplifies GenerateJitCallbackTrampolines.
// For TSAN, we always go to the runtime so TSAN is aware of the release
// semantics of entering the safepoint.
Register addr = RA;
ASSERT(addr != state);
Label slow_path, done, retry;
if (FLAG_use_slow_path || FLAG_target_thread_sanitizer) {
j(&slow_path, Assembler::kNearJump);
}
addi(addr, THR, target::Thread::safepoint_state_offset());
Bind(&retry);
lr(state, Address(addr, 0));
subi(state, state, target::Thread::native_safepoint_state_unacquired());
bnez(state, &slow_path, Assembler::kNearJump);
li(state, target::Thread::native_safepoint_state_acquired());
sc(state, state, Address(addr, 0));
beqz(state, &done, Assembler::kNearJump); // 0 means sc was successful.
if (!FLAG_use_slow_path && !FLAG_target_thread_sanitizer) {
j(&retry, Assembler::kNearJump);
}
Bind(&slow_path);
lx(addr, Address(THR, target::Thread::enter_safepoint_stub_offset()));
lx(addr, FieldAddress(addr, target::Code::entry_point_offset()));
jalr(addr);
Bind(&done);
}
void Assembler::ExitFullSafepoint(Register state) {
// We generate the same number of instructions whether or not the slow-path is
// forced, for consistency with EnterFullSafepoint.
// For TSAN, we always go to the runtime so TSAN is aware of the acquire
// semantics of leaving the safepoint.
Register addr = RA;
ASSERT(addr != state);
Label slow_path, done, retry;
if (FLAG_use_slow_path || FLAG_target_thread_sanitizer) {
j(&slow_path, Assembler::kNearJump);
}
addi(addr, THR, target::Thread::safepoint_state_offset());
Bind(&retry);
lr(state, Address(addr, 0));
subi(state, state, target::Thread::native_safepoint_state_acquired());
bnez(state, &slow_path, Assembler::kNearJump);
li(state, target::Thread::native_safepoint_state_unacquired());
sc(state, state, Address(addr, 0));
beqz(state, &done, Assembler::kNearJump); // 0 means sc was successful.
if (!FLAG_use_slow_path && !FLAG_target_thread_sanitizer) {
j(&retry, Assembler::kNearJump);
}
Bind(&slow_path);
lx(addr, Address(THR, target::Thread::exit_safepoint_stub_offset()));
lx(addr, FieldAddress(addr, target::Code::entry_point_offset()));
jalr(addr);
Bind(&done);
}
void Assembler::CheckFpSpDist(intptr_t fp_sp_dist) {
ASSERT(fp_sp_dist <= 0);
#if defined(DEBUG)
Label ok;
Comment("CheckFpSpDist");
sub(TMP, SP, FP);
CompareImmediate(TMP, fp_sp_dist);
BranchIf(EQ, &ok, compiler::Assembler::kNearJump);
ebreak();
Bind(&ok);
#endif
}
void Assembler::CheckCodePointer() {
#ifdef DEBUG
if (!FLAG_check_code_pointer) {
return;
}
Comment("CheckCodePointer");
Label cid_ok, instructions_ok;
CompareClassId(CODE_REG, kCodeCid, TMP);
BranchIf(EQ, &cid_ok, kNearJump);
ebreak();
Bind(&cid_ok);
const intptr_t entry_offset =
CodeSize() + target::Instructions::HeaderSize() - kHeapObjectTag;
intx_t imm = -entry_offset;
intx_t lo = ImmLo(imm);
intx_t hi = ImmHi(imm);
auipc(TMP, hi);
addi(TMP, TMP, lo);
lx(TMP2, FieldAddress(CODE_REG, target::Code::instructions_offset()));
beq(TMP, TMP2, &instructions_ok, kNearJump);
ebreak();
Bind(&instructions_ok);
#endif
}
void Assembler::RestoreCodePointer() {
lx(CODE_REG,
Address(FP, target::frame_layout.code_from_fp * target::kWordSize));
CheckCodePointer();
}
void Assembler::RestorePoolPointer() {
if (FLAG_precompiled_mode) {
lx(PP, Address(THR, target::Thread::global_object_pool_offset()));
} else {
lx(PP, Address(FP, target::frame_layout.code_from_fp * target::kWordSize));
lx(PP, FieldAddress(PP, target::Code::object_pool_offset()));
}
subi(PP, PP, kHeapObjectTag); // Pool in PP is untagged!
}
void Assembler::RestorePinnedRegisters() {
lx(WRITE_BARRIER_STATE,
Address(THR, target::Thread::write_barrier_mask_offset()));
lx(NULL_REG, Address(THR, target::Thread::object_null_offset()));
// Our write barrier usually uses mask-and-test,
// 01b6f6b3 and tmp, tmp, mask
// c689 beqz tmp, +10
// but on RISC-V compare-and-branch is shorter,
// 00ddd663 ble tmp, wbs, +12
//
// TMP bit 4+ = 0
// TMP bit 3 = object is old-and-not-remembered AND value is new (genr bit)
// TMP bit 2 = object is old AND value is old-and-not-marked (incr bit)
// TMP bit 1 = garbage
// TMP bit 0 = garbage
//
// Thread::wbm | WRITE_BARRIER_STATE | TMP/combined headers | result
// generational only
// 0b1000 0b0111 0b11xx impossible
// 0b10xx call stub
// 0b01xx skip
// 0b00xx skip
// generational and incremental
// 0b1100 0b0011 0b11xx impossible
// 0b10xx call stub
// 0b01xx call stub
// 0b00xx skip
xori(WRITE_BARRIER_STATE, WRITE_BARRIER_STATE,
(target::UntaggedObject::kGenerationalBarrierMask << 1) - 1);
// Generational bit must be higher than incremental bit, with no other bits
// between.
ASSERT(target::UntaggedObject::kGenerationalBarrierMask ==
(target::UntaggedObject::kIncrementalBarrierMask << 1));
// Other header bits must be lower.
ASSERT(target::UntaggedObject::kIncrementalBarrierMask >
target::UntaggedObject::kCanonicalBit);
ASSERT(target::UntaggedObject::kIncrementalBarrierMask >
target::UntaggedObject::kCardRememberedBit);
}
void Assembler::SetupGlobalPoolAndDispatchTable() {
ASSERT(FLAG_precompiled_mode);
lx(PP, Address(THR, target::Thread::global_object_pool_offset()));
subi(PP, PP, kHeapObjectTag); // Pool in PP is untagged!
lx(DISPATCH_TABLE_REG,
Address(THR, target::Thread::dispatch_table_array_offset()));
}
void Assembler::EnterDartFrame(intptr_t frame_size, Register new_pp) {
ASSERT(!constant_pool_allowed());
if (!IsITypeImm(frame_size + 4 * target::kWordSize)) {
EnterDartFrame(0, new_pp);
AddImmediate(SP, SP, -frame_size);
return;
}
// N.B. The ordering here is important. We must never write beyond SP or
// it can be clobbered by a signal handler.
if (FLAG_precompiled_mode) {
subi(SP, SP, frame_size + 2 * target::kWordSize);
sx(RA, Address(SP, frame_size + 1 * target::kWordSize));
sx(FP, Address(SP, frame_size + 0 * target::kWordSize));
addi(FP, SP, frame_size + 2 * target::kWordSize);
} else {
subi(SP, SP, frame_size + 4 * target::kWordSize);
sx(RA, Address(SP, frame_size + 3 * target::kWordSize));
sx(FP, Address(SP, frame_size + 2 * target::kWordSize));
sx(CODE_REG, Address(SP, frame_size + 1 * target::kWordSize));
addi(PP, PP, kHeapObjectTag);
sx(PP, Address(SP, frame_size + 0 * target::kWordSize));
addi(FP, SP, frame_size + 4 * target::kWordSize);
if (new_pp == kNoRegister) {
LoadPoolPointer();
} else {
mv(PP, new_pp);
}
}
set_constant_pool_allowed(true);
}
// On entry to a function compiled for OSR, the caller's frame pointer, the
// stack locals, and any copied parameters are already in place. The frame
// pointer is already set up. The PC marker is not correct for the
// optimized function and there may be extra space for spill slots to
// allocate. We must also set up the pool pointer for the function.
void Assembler::EnterOsrFrame(intptr_t extra_size, Register new_pp) {
ASSERT(!constant_pool_allowed());
Comment("EnterOsrFrame");
RestoreCodePointer();
LoadPoolPointer();
if (extra_size > 0) {
AddImmediate(SP, -extra_size);
}
}
void Assembler::LeaveDartFrame() {
// N.B. The ordering here is important. We must never read beyond SP or
// it may have already been clobbered by a signal handler.
if (!FLAG_precompiled_mode) {
lx(PP, Address(FP, target::frame_layout.saved_caller_pp_from_fp *
target::kWordSize));
subi(PP, PP, kHeapObjectTag);
}
set_constant_pool_allowed(false);
subi(SP, FP, 2 * target::kWordSize);
// Update RA first so the profiler can identify the frame as the entry frame
// after FP is updated but before the return instruction.
lx(RA, Address(SP, 1 * target::kWordSize));
lx(FP, Address(SP, 0 * target::kWordSize));
addi(SP, SP, 2 * target::kWordSize);
}
void Assembler::LeaveDartFrame(intptr_t fp_sp_dist) {
intptr_t pp_offset =
target::frame_layout.saved_caller_pp_from_fp * target::kWordSize -
fp_sp_dist;
intptr_t fp_offset =
target::frame_layout.saved_caller_fp_from_fp * target::kWordSize -
fp_sp_dist;
intptr_t ra_offset =
target::frame_layout.saved_caller_pc_from_fp * target::kWordSize -
fp_sp_dist;
if (!IsITypeImm(pp_offset) || !IsITypeImm(fp_offset) ||
!IsITypeImm(ra_offset)) {
// Shorter to update SP twice than generate large immediates.
LeaveDartFrame();
return;
}
if (!FLAG_precompiled_mode) {
lx(PP, Address(SP, pp_offset));
subi(PP, PP, kHeapObjectTag);
}
set_constant_pool_allowed(false);
lx(FP, Address(SP, fp_offset));
lx(RA, Address(SP, ra_offset));
addi(SP, SP, -fp_sp_dist);
}
void Assembler::CallRuntime(const RuntimeEntry& entry,
intptr_t argument_count) {
ASSERT(!entry.is_leaf());
// Argument count is not checked here, but in the runtime entry for a more
// informative error message.
lx(T5, compiler::Address(THR, entry.OffsetFromThread()));
li(T4, argument_count);
Call(Address(THR, target::Thread::call_to_runtime_entry_point_offset()));
}
static const RegisterSet kRuntimeCallSavedRegisters(kDartVolatileCpuRegs,
kAbiVolatileFpuRegs);
#define __ assembler_->
LeafRuntimeScope::LeafRuntimeScope(Assembler* assembler,
intptr_t frame_size,
bool preserve_registers)
: assembler_(assembler), preserve_registers_(preserve_registers) {
// N.B. The ordering here is important. We must never write beyond SP or
// it can be clobbered by a signal handler.
__ subi(SP, SP, 4 * target::kWordSize);
__ sx(RA, Address(SP, 3 * target::kWordSize));
__ sx(FP, Address(SP, 2 * target::kWordSize));
__ sx(CODE_REG, Address(SP, 1 * target::kWordSize));
__ sx(PP, Address(SP, 0 * target::kWordSize));
__ addi(FP, SP, 4 * target::kWordSize);
if (preserve_registers) {
__ PushRegisters(kRuntimeCallSavedRegisters);
} else {
// Or no reason to save above.
COMPILE_ASSERT(!IsAbiPreservedRegister(CODE_REG));
COMPILE_ASSERT(!IsAbiPreservedRegister(PP));
// Or would need to save above.
COMPILE_ASSERT(IsCalleeSavedRegister(THR));
COMPILE_ASSERT(IsCalleeSavedRegister(NULL_REG));
COMPILE_ASSERT(IsCalleeSavedRegister(WRITE_BARRIER_STATE));
COMPILE_ASSERT(IsCalleeSavedRegister(DISPATCH_TABLE_REG));
}
__ ReserveAlignedFrameSpace(frame_size);
}
void LeafRuntimeScope::Call(const RuntimeEntry& entry,
intptr_t argument_count) {
ASSERT(argument_count == entry.argument_count());
__ Load(TMP2, compiler::Address(THR, entry.OffsetFromThread()));
__ sx(TMP2, compiler::Address(THR, target::Thread::vm_tag_offset()));
__ jalr(TMP2);
__ LoadImmediate(TMP2, VMTag::kDartTagId);
__ sx(TMP2, compiler::Address(THR, target::Thread::vm_tag_offset()));
}
LeafRuntimeScope::~LeafRuntimeScope() {
if (preserve_registers_) {
__ subi(SP, FP,
kRuntimeCallSavedRegisters.SpillSize() + 4 * target::kWordSize);
__ PopRegisters(kRuntimeCallSavedRegisters);
}
__ subi(SP, FP, 4 * target::kWordSize);
__ lx(PP, Address(SP, 0 * target::kWordSize));
__ lx(CODE_REG, Address(SP, 1 * target::kWordSize));
__ lx(FP, Address(SP, 2 * target::kWordSize));
__ lx(RA, Address(SP, 3 * target::kWordSize));
__ addi(SP, SP, 4 * target::kWordSize);
}
#undef __
void Assembler::EnterCFrame(intptr_t frame_space) {
// Already saved.
COMPILE_ASSERT(IsCalleeSavedRegister(THR));
COMPILE_ASSERT(IsCalleeSavedRegister(NULL_REG));
COMPILE_ASSERT(IsCalleeSavedRegister(WRITE_BARRIER_STATE));
COMPILE_ASSERT(IsCalleeSavedRegister(DISPATCH_TABLE_REG));
// Need to save.
COMPILE_ASSERT(!IsCalleeSavedRegister(PP));
// N.B. The ordering here is important. We must never read beyond SP or
// it may have already been clobbered by a signal handler.
subi(SP, SP, frame_space + 3 * target::kWordSize);
sx(RA, Address(SP, frame_space + 2 * target::kWordSize));
sx(FP, Address(SP, frame_space + 1 * target::kWordSize));
sx(PP, Address(SP, frame_space + 0 * target::kWordSize));
addi(FP, SP, frame_space + 3 * target::kWordSize);
const intptr_t kAbiStackAlignment = 16; // For both 32 and 64 bit.
andi(SP, SP, ~(kAbiStackAlignment - 1));
}
void Assembler::LeaveCFrame() {
// N.B. The ordering here is important. We must never read beyond SP or
// it may have already been clobbered by a signal handler.
subi(SP, FP, 3 * target::kWordSize);
lx(PP, Address(SP, 0 * target::kWordSize));
lx(FP, Address(SP, 1 * target::kWordSize));
lx(RA, Address(SP, 2 * target::kWordSize));
addi(SP, SP, 3 * target::kWordSize);
}
// A0: Receiver
// S5: ICData entry array
// PP: Caller's PP (preserved)
void Assembler::MonomorphicCheckedEntryJIT() {
has_monomorphic_entry_ = true;
const intptr_t saved_far_branch_level = far_branch_level();
set_far_branch_level(0);
const intptr_t start = CodeSize();
Label immediate, miss;
Bind(&miss);
lx(TMP, Address(THR, target::Thread::switchable_call_miss_entry_offset()));
jr(TMP);
Comment("MonomorphicCheckedEntry");
ASSERT_EQUAL(CodeSize() - start,
target::Instructions::kMonomorphicEntryOffsetJIT);
Register entries_reg = IC_DATA_REG; // Contains ICData::entries().
const intptr_t cid_offset = target::Array::element_offset(0);
const intptr_t count_offset = target::Array::element_offset(1);
ASSERT(A1 != PP);
ASSERT(A1 != entries_reg);
ASSERT(A1 != CODE_REG);
lx(TMP, FieldAddress(entries_reg, cid_offset));
LoadTaggedClassIdMayBeSmi(A1, A0);
bne(TMP, A1, &miss, kNearJump);
lx(TMP, FieldAddress(entries_reg, count_offset));
addi(TMP, TMP, target::ToRawSmi(1));
sx(TMP, FieldAddress(entries_reg, count_offset));
li(ARGS_DESC_REG, 0); // GC-safe for OptimizeInvokedFunction
// Fall through to unchecked entry.
ASSERT_EQUAL(CodeSize() - start,
target::Instructions::kPolymorphicEntryOffsetJIT);
set_far_branch_level(saved_far_branch_level);
}
// A0 receiver, S5 guarded cid as Smi.
// Preserve S4 (ARGS_DESC_REG), not required today, but maybe later.
// PP: Caller's PP (preserved)
void Assembler::MonomorphicCheckedEntryAOT() {
has_monomorphic_entry_ = true;
intptr_t saved_far_branch_level = far_branch_level();
set_far_branch_level(0);
const intptr_t start = CodeSize();
Label immediate, miss;
Bind(&miss);
lx(TMP, Address(THR, target::Thread::switchable_call_miss_entry_offset()));
jr(TMP);
Comment("MonomorphicCheckedEntry");
ASSERT_EQUAL(CodeSize() - start,
target::Instructions::kMonomorphicEntryOffsetAOT);
LoadClassId(TMP, A0);
SmiTag(TMP);
bne(S5, TMP, &miss, kNearJump);
// Fall through to unchecked entry.
ASSERT_EQUAL(CodeSize() - start,
target::Instructions::kPolymorphicEntryOffsetAOT);
set_far_branch_level(saved_far_branch_level);
}
void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
has_monomorphic_entry_ = true;
while (CodeSize() < target::Instructions::kMonomorphicEntryOffsetJIT) {
ebreak();
}
j(label);
while (CodeSize() < target::Instructions::kPolymorphicEntryOffsetJIT) {
ebreak();
}
}
void Assembler::CombineHashes(Register hash, Register other) {
#if XLEN >= 64
// hash += other_hash
addw(hash, hash, other);
// hash += hash << 10
slliw(other, hash, 10);
addw(hash, hash, other);
// hash ^= hash >> 6
srliw(other, hash, 6);
xor_(hash, hash, other);
#else
// hash += other_hash
add(hash, hash, other);
// hash += hash << 10
slli(other, hash, 10);
add(hash, hash, other);
// hash ^= hash >> 6
srli(other, hash, 6);
xor_(hash, hash, other);
#endif
}
void Assembler::FinalizeHashForSize(intptr_t bit_size,
Register hash,
Register scratch) {
ASSERT(bit_size > 0); // Can't avoid returning 0 if there are no hash bits!
// While any 32-bit hash value fits in X bits, where X > 32, the caller may
// reasonably expect that the returned values fill the entire bit space.
ASSERT(bit_size <= kBitsPerInt32);
ASSERT(scratch != kNoRegister);
#if XLEN >= 64
// hash += hash << 3;
slliw(scratch, hash, 3);
addw(hash, hash, scratch);
// hash ^= hash >> 11; // Logical shift, unsigned hash.
srliw(scratch, hash, 11);
xor_(hash, hash, scratch);
// hash += hash << 15;
slliw(scratch, hash, 15);
addw(hash, hash, scratch);
#else
// hash += hash << 3;
slli(scratch, hash, 3);
add(hash, hash, scratch);
// hash ^= hash >> 11; // Logical shift, unsigned hash.
srli(scratch, hash, 11);
xor_(hash, hash, scratch);
// hash += hash << 15;
slli(scratch, hash, 15);
add(hash, hash, scratch);
#endif
// Size to fit.
if (bit_size < kBitsPerInt32) {
AndImmediate(hash, hash, Utils::NBitMask(bit_size));
}
// return (hash == 0) ? 1 : hash;
seqz(scratch, hash);
add(hash, hash, scratch);
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(Register cid,
Label* trace,
Register temp_reg,
JumpDistance distance) {
LoadIsolateGroup(temp_reg);
lx(temp_reg, Address(temp_reg, target::IsolateGroup::class_table_offset()));
lx(temp_reg,
Address(temp_reg,
target::ClassTable::allocation_tracing_state_table_offset()));
add(temp_reg, temp_reg, cid);
LoadFromOffset(temp_reg, temp_reg,
target::ClassTable::AllocationTracingStateSlotOffsetFor(0),
kUnsignedByte);
bnez(temp_reg, trace);
}
void Assembler::MaybeTraceAllocation(intptr_t cid,
Label* trace,
Register temp_reg,
JumpDistance distance) {
ASSERT(cid > 0);
LoadIsolateGroup(temp_reg);
lx(temp_reg, Address(temp_reg, target::IsolateGroup::class_table_offset()));
lx(temp_reg,
Address(temp_reg,
target::ClassTable::allocation_tracing_state_table_offset()));
LoadFromOffset(temp_reg, temp_reg,
target::ClassTable::AllocationTracingStateSlotOffsetFor(cid),
kUnsignedByte);
bnez(temp_reg, trace);
}
#endif // !PRODUCT
void Assembler::TryAllocateObject(intptr_t cid,
intptr_t instance_size,
Label* failure,
JumpDistance distance,
Register instance_reg,
Register temp_reg) {
ASSERT(failure != nullptr);
ASSERT(instance_size != 0);
ASSERT(instance_reg != temp_reg);
ASSERT(temp_reg != kNoRegister);
ASSERT(Utils::IsAligned(instance_size,
target::ObjectAlignment::kObjectAlignment));
if (FLAG_inline_alloc &&
target::Heap::IsAllocatableInNewSpace(instance_size)) {
// If this allocation is traced, program will jump to failure path
// (i.e. the allocation stub) which will allocate the object and trace the
// allocation call site.
NOT_IN_PRODUCT(MaybeTraceAllocation(cid, failure, temp_reg));
lx(instance_reg, Address(THR, target::Thread::top_offset()));
lx(temp_reg, Address(THR, target::Thread::end_offset()));
// instance_reg: current top (next object start).
// temp_reg: heap end
// TODO(koda): Protect against unsigned overflow here.
AddImmediate(instance_reg, instance_size);
// instance_reg: potential top (next object start).
// fail if heap end unsigned less than or equal to new heap top.
bleu(temp_reg, instance_reg, failure, distance);
CheckAllocationCanary(instance_reg, temp_reg);
// Successfully allocated the object, now update temp to point to
// next object start and store the class in the class field of object.
sx(instance_reg, Address(THR, target::Thread::top_offset()));
// Move instance_reg back to the start of the object and tag it.
AddImmediate(instance_reg, -instance_size + kHeapObjectTag);
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
LoadImmediate(temp_reg, tags);
InitializeHeader(temp_reg, instance_reg);
} else {
j(failure, distance);
}
}
void Assembler::TryAllocateArray(intptr_t cid,
intptr_t instance_size,
Label* failure,
Register instance,
Register end_address,
Register temp1,
Register temp2) {
if (FLAG_inline_alloc &&
target::Heap::IsAllocatableInNewSpace(instance_size)) {
// If this allocation is traced, program will jump to failure path
// (i.e. the allocation stub) which will allocate the object and trace the
// allocation call site.
NOT_IN_PRODUCT(MaybeTraceAllocation(cid, failure, temp1));
// Potential new object start.
lx(instance, Address(THR, target::Thread::top_offset()));
AddImmediate(end_address, instance, instance_size);
bltu(end_address, instance, failure); // Fail on unsigned overflow.
// Check if the allocation fits into the remaining space.
// instance: potential new object start.
// end_address: potential next object start.
lx(temp2, Address(THR, target::Thread::end_offset()));
bgeu(end_address, temp2, failure);
CheckAllocationCanary(instance, temp2);
// Successfully allocated the object(s), now update top to point to
// next object start and initialize the object.
sx(end_address, Address(THR, target::Thread::top_offset()));
addi(instance, instance, kHeapObjectTag);
NOT_IN_PRODUCT(LoadImmediate(temp2, instance_size));
// Initialize the tags.
// instance: new object start as a tagged pointer.
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
LoadImmediate(temp2, tags);
InitializeHeader(temp2, instance);
} else {
j(failure);
}
}
void Assembler::CopyMemoryWords(Register src,
Register dst,
Register size,
Register temp) {
Label loop, done;
beqz(size, &done, kNearJump);
Bind(&loop);
lx(temp, Address(src));
addi(src, src, target::kWordSize);
sx(temp, Address(dst));
addi(dst, dst, target::kWordSize);
subi(size, size, target::kWordSize);
bnez(size, &loop, kNearJump);
Bind(&done);
}
void Assembler::GenerateUnRelocatedPcRelativeCall(intptr_t offset_into_target) {
// JAL only has a +/- 1MB range. AUIPC+JALR has a +/- 2GB range.
intx_t lo = ImmLo(offset_into_target);
intx_t hi = ImmHi(offset_into_target);
auipc(RA, hi);
jalr_fixed(RA, RA, lo);
}
void Assembler::GenerateUnRelocatedPcRelativeTailCall(
intptr_t offset_into_target) {
// J only has a +/- 1MB range. AUIPC+JR has a +/- 2GB range.
intx_t lo = ImmLo(offset_into_target);
intx_t hi = ImmHi(offset_into_target);
auipc(TMP, hi);
jalr_fixed(ZR, TMP, lo);
}
bool Assembler::AddressCanHoldConstantIndex(const Object& constant,
bool is_external,
intptr_t cid,
intptr_t index_scale) {
if (!IsSafeSmi(constant)) return false;
const int64_t index = target::SmiValue(constant);
const int64_t offset = index * index_scale + HeapDataOffset(is_external, cid);
if (IsITypeImm(offset)) {
ASSERT(IsSTypeImm(offset));
return true;
}
return false;
}
Address Assembler::ElementAddressForIntIndex(bool is_external,
intptr_t cid,
intptr_t index_scale,
Register array,
intptr_t index) const {
const int64_t offset = index * index_scale + HeapDataOffset(is_external, cid);
ASSERT(Utils::IsInt(32, offset));
return Address(array, static_cast<int32_t>(offset));
}
void Assembler::ComputeElementAddressForIntIndex(Register address,
bool is_external,
intptr_t cid,
intptr_t index_scale,
Register array,
intptr_t index) {
const int64_t offset = index * index_scale + HeapDataOffset(is_external, cid);
AddImmediate(address, array, offset);
}
Address Assembler::ElementAddressForRegIndex(bool is_external,
intptr_t cid,
intptr_t index_scale,
bool index_unboxed,
Register array,
Register index,
Register temp) {
// If unboxed, index is expected smi-tagged, (i.e, LSL 1) for all arrays.
const intptr_t boxing_shift = index_unboxed ? 0 : -kSmiTagShift;
const intptr_t shift = Utils::ShiftForPowerOfTwo(index_scale) + boxing_shift;
const int32_t offset = HeapDataOffset(is_external, cid);
ASSERT(array != temp);
ASSERT(index != temp);
AddShifted(temp, array, index, shift);
return Address(temp, offset);
}
void Assembler::ComputeElementAddressForRegIndex(Register address,
bool is_external,
intptr_t cid,
intptr_t index_scale,
bool index_unboxed,
Register array,
Register index) {
// If unboxed, index is expected smi-tagged, (i.e, LSL 1) for all arrays.
const intptr_t boxing_shift = index_unboxed ? 0 : -kSmiTagShift;
const intptr_t shift = Utils::ShiftForPowerOfTwo(index_scale) + boxing_shift;
const int32_t offset = HeapDataOffset(is_external, cid);
ASSERT(array != address);
ASSERT(index != address);
AddShifted(address, array, index, shift);
if (offset != 0) {
AddImmediate(address, address, offset);
}
}
void Assembler::LoadStaticFieldAddress(Register address,
Register field,
Register scratch,
bool is_shared) {
LoadCompressedSmiFieldFromOffset(
scratch, field, target::Field::host_offset_or_field_id_offset());
const intptr_t field_table_offset =
is_shared ? compiler::target::Thread::shared_field_table_values_offset()
: compiler::target::Thread::field_table_values_offset();
LoadMemoryValue(address, THR, static_cast<int32_t>(field_table_offset));
slli(scratch, scratch, target::kWordSizeLog2 - kSmiTagShift);
add(address, address, scratch);
}
void Assembler::LoadFieldAddressForRegOffset(Register address,
Register instance,
Register offset_in_words_as_smi) {
AddShifted(address, instance, offset_in_words_as_smi,
target::kWordSizeLog2 - kSmiTagShift);
addi(address, address, -kHeapObjectTag);
}
// Note: the function never clobbers TMP, TMP2 scratch registers.
void Assembler::LoadObjectHelper(
Register dst,
const Object& object,
bool is_unique,
ObjectPoolBuilderEntry::SnapshotBehavior snapshot_behavior) {
ASSERT(IsOriginalObject(object));
// `is_unique == true` effectively means object has to be patchable.
// (even if the object is null)
if (!is_unique) {
if (IsSameObject(compiler::NullObject(), object)) {
mv(dst, NULL_REG);
return;
}
if (IsSameObject(CastHandle<Object>(compiler::TrueObject()), object)) {
addi(dst, NULL_REG, kTrueOffsetFromNull);
return;
}
if (IsSameObject(CastHandle<Object>(compiler::FalseObject()), object)) {
addi(dst, NULL_REG, kFalseOffsetFromNull);
return;
}
word offset = 0;
if (target::CanLoadFromThread(object, &offset)) {
lx(dst, Address(THR, offset));
return;
}
if (target::IsSmi(object)) {
LoadImmediate(dst, target::ToRawSmi(object));
return;
}
}
RELEASE_ASSERT(CanLoadFromObjectPool(object));
const intptr_t index =
is_unique
? object_pool_builder().AddObject(
object, ObjectPoolBuilderEntry::kPatchable, snapshot_behavior)
: object_pool_builder().FindObject(
object, ObjectPoolBuilderEntry::kNotPatchable,
snapshot_behavior);
LoadWordFromPoolIndex(dst, index);
}
void Assembler::AddImmediateBranchOverflow(Register rd,
Register rs1,
intx_t imm,
Label* overflow) {
ASSERT(rd != TMP2);
if (rd == rs1) {
mv(TMP2, rs1);
AddImmediate(rd, rs1, imm);
if (imm > 0) {
blt(rd, TMP2, overflow);
} else if (imm < 0) {
bgt(rd, TMP2, overflow);
}
} else {
AddImmediate(rd, rs1, imm);
if (imm > 0) {
blt(rd, rs1, overflow);
} else if (imm < 0) {
bgt(rd, rs1, overflow);
}
}
}
void Assembler::SubtractImmediateBranchOverflow(Register rd,
Register rs1,
intx_t imm,
Label* overflow) {
// TODO(riscv): Incorrect for MIN_INTX_T!
AddImmediateBranchOverflow(rd, rs1, -imm, overflow);
}
void Assembler::MultiplyImmediateBranchOverflow(Register rd,
Register rs1,
intx_t imm,
Label* overflow) {
ASSERT(rd != TMP);
ASSERT(rd != TMP2);
ASSERT(rs1 != TMP);
ASSERT(rs1 != TMP2);
LoadImmediate(TMP2, imm);
// Macro-op fusion: when both products are needed, the recommended sequence
// is mulh first.
mulh(TMP, rs1, TMP2);
mul(rd, rs1, TMP2);
srai(TMP2, rd, XLEN - 1);
bne(TMP, TMP2, overflow);
}
void Assembler::AddBranchOverflow(Register rd,
Register rs1,
Register rs2,
Label* overflow) {
ASSERT(rd != TMP);
ASSERT(rd != TMP2);
ASSERT(rs1 != TMP);
ASSERT(rs1 != TMP2);
ASSERT(rs2 != TMP);
ASSERT(rs2 != TMP2);
if ((rd == rs1) && (rd == rs2)) {
ASSERT(rs1 == rs2);
mv(TMP, rs1);
add(rd, rs1, rs2); // rs1, rs2 destroyed
xor_(TMP, TMP, rd); // TMP negative if sign changed
bltz(TMP, overflow);
} else if (rs1 == rs2) {
ASSERT(rd != rs1);
ASSERT(rd != rs2);
add(rd, rs1, rs2);
xor_(TMP, rd, rs1); // TMP negative if sign changed
bltz(TMP, overflow);
} else if (rd == rs1) {
ASSERT(rs1 != rs2);
slti(TMP, rs1, 0);
add(rd, rs1, rs2); // rs1 destroyed
slt(TMP2, rd, rs2);
bne(TMP, TMP2, overflow);
} else if (rd == rs2) {
ASSERT(rs1 != rs2);
slti(TMP, rs2, 0);
add(rd, rs1, rs2); // rs2 destroyed
slt(TMP2, rd, rs1);
bne(TMP, TMP2, overflow);
} else {
add(rd, rs1, rs2);
slti(TMP, rs2, 0);
slt(TMP2, rd, rs1);
bne(TMP, TMP2, overflow);
}
}
void Assembler::SubtractBranchOverflow(Register rd,
Register rs1,
Register rs2,
Label* overflow) {
ASSERT(rd != TMP);
ASSERT(rd != TMP2);
ASSERT(rs1 != TMP);
ASSERT(rs1 != TMP2);
ASSERT(rs2 != TMP);
ASSERT(rs2 != TMP2);
if ((rd == rs1) && (rd == rs2)) {
ASSERT(rs1 == rs2);
mv(TMP, rs1);
sub(rd, rs1, rs2); // rs1, rs2 destroyed
xor_(TMP, TMP, rd); // TMP negative if sign changed
bltz(TMP, overflow);
} else if (rs1 == rs2) {
ASSERT(rd != rs1);
ASSERT(rd != rs2);
sub(rd, rs1, rs2);
xor_(TMP, rd, rs1); // TMP negative if sign changed
bltz(TMP, overflow);
} else if (rd == rs1) {
ASSERT(rs1 != rs2);
slti(TMP, rs1, 0);
sub(rd, rs1, rs2); // rs1 destroyed
slt(TMP2, rd, rs2);
bne(TMP, TMP2, overflow);
} else if (rd == rs2) {
ASSERT(rs1 != rs2);
slti(TMP, rs2, 0);
sub(rd, rs1, rs2); // rs2 destroyed
slt(TMP2, rd, rs1);
bne(TMP, TMP2, overflow);
} else {
sub(rd, rs1, rs2);
slti(TMP, rs2, 0);
slt(TMP2, rs1, rd);
bne(TMP, TMP2, overflow);
}
}
void Assembler::MultiplyBranchOverflow(Register rd,
Register rs1,
Register rs2,
Label* overflow) {
ASSERT(rd != TMP);
ASSERT(rd != TMP2);
ASSERT(rs1 != TMP);
ASSERT(rs1 != TMP2);
ASSERT(rs2 != TMP);
ASSERT(rs2 != TMP2);
// Macro-op fusion: when both products are needed, the recommended sequence
// is mulh first.
mulh(TMP, rs1, rs2);
mul(rd, rs1, rs2);
srai(TMP2, rd, XLEN - 1);
bne(TMP, TMP2, overflow);
}
void Assembler::CountLeadingZeroes(Register rd, Register rs) {
if (Supports(RV_Zbb)) {
clz(rd, rs);
return;
}
// n = XLEN
// y = x >>32; if (y != 0) { n = n - 32; x = y; }
// y = x >>16; if (y != 0) { n = n - 16; x = y; }
// y = x >> 8; if (y != 0) { n = n - 8; x = y; }
// y = x >> 4; if (y != 0) { n = n - 4; x = y; }
// y = x >> 2; if (y != 0) { n = n - 2; x = y; }
// y = x >> 1; if (y != 0) { return n - 2; }
// return n - x;
Label l0, l1, l2, l3, l4, l5;
li(TMP2, XLEN);
#if XLEN == 64
srli(TMP, rs, 32);
beqz(TMP, &l0, Assembler::kNearJump);
subi(TMP2, TMP2, 32);
mv(rs, TMP);
Bind(&l0);
#endif
srli(TMP, rs, 16);
beqz(TMP, &l1, Assembler::kNearJump);
subi(TMP2, TMP2, 16);
mv(rs, TMP);
Bind(&l1);
srli(TMP, rs, 8);
beqz(TMP, &l2, Assembler::kNearJump);
subi(TMP2, TMP2, 8);
mv(rs, TMP);
Bind(&l2);
srli(TMP, rs, 4);
beqz(TMP, &l3, Assembler::kNearJump);
subi(TMP2, TMP2, 4);
mv(rs, TMP);
Bind(&l3);
srli(TMP, rs, 2);
beqz(TMP, &l4, Assembler::kNearJump);
subi(TMP2, TMP2, 2);
mv(rs, TMP);
Bind(&l4);
srli(TMP, rs, 1);
sub(rd, TMP2, rs);
beqz(TMP, &l5, Assembler::kNearJump);
subi(rd, TMP2, 2);
Bind(&l5);
}
void Assembler::RangeCheck(Register value,
Register temp,
intptr_t low,
intptr_t high,
RangeCheckCondition condition,
Label* target) {
auto cc = condition == kIfInRange ? LS : HI;
Register to_check = temp != kNoRegister ? temp : value;
AddImmediate(to_check, value, -low);
CompareImmediate(to_check, high - low);
BranchIf(cc, target);
}
} // namespace compiler
} // namespace dart
#endif // defined(TARGET_ARCH_RISCV)