| // Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| #include "vm/compiler/ffi/native_calling_convention.h" |
| |
| #include "vm/compiler/ffi/native_location.h" |
| #include "vm/compiler/ffi/native_type.h" |
| #include "vm/zone_text_buffer.h" |
| |
| #if !defined(FFI_UNIT_TESTS) |
| #include "vm/cpu.h" |
| #endif |
| |
| namespace dart { |
| |
| namespace compiler { |
| |
| namespace ffi { |
| |
| const intptr_t kNoFpuRegister = -1; |
| |
| #if !defined(FFI_UNIT_TESTS) |
| // In Soft FP and vararg calls, floats and doubles get passed in integer |
| // registers. |
| static bool SoftFpAbi(bool has_varargs, bool is_result) { |
| #if defined(TARGET_ARCH_ARM) |
| if (has_varargs) { |
| return true; |
| } |
| return !TargetCPUFeatures::hardfp_supported(); |
| #elif defined(TARGET_ARCH_ARM64) && defined(DART_TARGET_OS_WINDOWS) |
| return has_varargs && !is_result; |
| #else |
| return false; |
| #endif |
| } |
| #else // !defined(FFI_UNIT_TESTS) |
| static bool SoftFpAbi(bool has_varargs, bool is_result) { |
| #if defined(TARGET_ARCH_ARM) && defined(DART_TARGET_OS_ANDROID) |
| return true; |
| #elif defined(TARGET_ARCH_ARM) |
| return has_varargs; |
| #elif defined(TARGET_ARCH_ARM64) && defined(DART_TARGET_OS_WINDOWS) |
| return has_varargs && !is_result; |
| #else |
| return false; |
| #endif |
| } |
| #endif // !defined(FFI_UNIT_TESTS) |
| |
| static const NativeType& ConvertFloatToInt(Zone* zone, const NativeType& type) { |
| ASSERT(type.IsFloat()); |
| if (type.SizeInBytes() == 4) { |
| return *new (zone) NativePrimitiveType(kInt32); |
| } |
| ASSERT(type.SizeInBytes() == 8); |
| return *new (zone) NativePrimitiveType(kInt64); |
| } |
| |
| // In Soft FP, floats are treated as 4 byte ints, and doubles as 8 byte ints. |
| static const NativeType& ConvertIfSoftFp(Zone* zone, |
| const NativeType& type, |
| bool has_varargs, |
| bool is_result = false) { |
| if (SoftFpAbi(has_varargs, is_result) && type.IsFloat()) { |
| return ConvertFloatToInt(zone, type); |
| } |
| return type; |
| } |
| |
| static PrimitiveType TypeForSize(intptr_t size) { |
| switch (size) { |
| case 8: |
| return kUint64; |
| case 7: |
| return kUint56; |
| case 6: |
| return kUint48; |
| case 5: |
| return kUint40; |
| case 4: |
| return kUint32; |
| case 3: |
| return kUint24; |
| case 2: |
| return kUint16; |
| case 1: |
| return kUint8; |
| default: |
| UNREACHABLE(); |
| return kVoid; |
| } |
| } |
| |
| // Represents the state of a stack frame going into a call, between allocations |
| // of argument locations. |
| class ArgumentAllocator : public ValueObject { |
| public: |
| explicit ArgumentAllocator(Zone* zone, bool has_varargs) |
| : has_varargs_(has_varargs), zone_(zone) {} |
| |
| const NativeLocation& AllocateArgumentVariadic(const NativeType& payload_type, |
| bool is_first_vararg = false, |
| bool is_vararg = false) { |
| #if defined(TARGET_ARCH_ARM64) && \ |
| (defined(DART_TARGET_OS_MACOS_IOS) || defined(DART_TARGET_OS_MACOS)) |
| if (is_first_vararg) { |
| // Block all registers. |
| BlockAllFpuRegisters(); |
| cpu_regs_used = CallingConventions::kNumArgRegs; |
| } |
| #endif |
| #if defined(TARGET_ARCH_RISCV64) || defined(TARGET_ARCH_RISCV32) |
| if (is_first_vararg) { |
| // Block all FPU registers. |
| BlockAllFpuRegisters(); |
| } |
| #endif |
| const auto& result = AllocateArgument(payload_type, is_vararg); |
| #if defined(TARGET_ARCH_X64) && defined(DART_TARGET_OS_WINDOWS) |
| if (has_varargs_) { |
| if (result.IsRegisters()) { |
| // If an integer register is used, block the corresponding xmm register. |
| ASSERT(CallingConventions::kArgumentIntRegXorFpuReg); |
| } else if (result.IsFpuRegisters()) { |
| // If an xmm register is used, also the corresponding integer register. |
| ASSERT(CallingConventions::kArgumentIntRegXorFpuReg); |
| const auto& fpu_reg_location = result.AsFpuRegisters(); |
| const FpuRegisterKind kind = kQuadFpuReg; |
| ASSERT(fpu_reg_location.fpu_reg_kind() == kind); |
| FpuRegister fpu_register = fpu_reg_location.fpu_reg(); |
| const intptr_t reg_index = fpu_register; |
| ASSERT(cpu_regs_used == reg_index + 1); |
| Register cpu_register = |
| CallingConventions::ArgumentRegisters[reg_index]; |
| const auto& container_type = ConvertFloatToInt(zone_, payload_type); |
| const auto& cpu_reg_location = *new (zone_) NativeRegistersLocation( |
| zone_, payload_type, container_type, cpu_register); |
| return *new (zone_) |
| BothNativeLocations(fpu_reg_location, cpu_reg_location); |
| } |
| } |
| #endif |
| return result; |
| } |
| |
| private: |
| const NativeLocation& AllocateArgument(const NativeType& payload_type, |
| |
| bool is_vararg = false) { |
| const auto& payload_type_converted = |
| ConvertIfSoftFp(zone_, payload_type, has_varargs_); |
| if (payload_type_converted.IsFloat()) { |
| return AllocateFloat(payload_type, is_vararg); |
| } |
| if (payload_type_converted.IsInt()) { |
| return AllocateInt(payload_type, is_vararg); |
| } |
| |
| // Compounds are laid out differently per ABI, so they are implemented |
| // per ABI. |
| // |
| // Compounds always have a PointerToMemory, Stack, or Multiple location, |
| // even if the parts of a compound fit in 1 cpu or fpu register it will |
| // be nested in a MultipleNativeLocations. |
| const NativeCompoundType& compound_type = payload_type.AsCompound(); |
| return AllocateCompound(compound_type, is_vararg, /*is_result*/ false); |
| } |
| |
| const NativeLocation& AllocateFloat(const NativeType& payload_type, |
| bool is_vararg) { |
| const auto kind = FpuRegKind(payload_type); |
| const intptr_t reg_index = FirstFreeFpuRegisterIndex(kind); |
| if (reg_index != kNoFpuRegister) { |
| AllocateFpuRegisterAtIndex(kind, reg_index); |
| if (CallingConventions::kArgumentIntRegXorFpuReg) { |
| cpu_regs_used++; |
| } |
| #if defined(TARGET_ARCH_ARM) |
| if (kind == kSingleFpuReg) { |
| return *new (zone_) |
| NativeFpuRegistersLocation(payload_type, payload_type, kind, |
| static_cast<SRegister>(reg_index)); |
| } |
| if (kind == kDoubleFpuReg) { |
| return *new (zone_) |
| NativeFpuRegistersLocation(payload_type, payload_type, kind, |
| static_cast<DRegister>(reg_index)); |
| } |
| #endif |
| ASSERT(kind == kQuadFpuReg); |
| FpuRegister reg = CallingConventions::FpuArgumentRegisters[reg_index]; |
| return *new (zone_) |
| NativeFpuRegistersLocation(payload_type, payload_type, reg); |
| } |
| |
| #if defined(TARGET_ARCH_RISCV64) |
| // After using up F registers, start bitcasting to X registers. |
| if (HasAvailableCpuRegisters(1)) { |
| const Register reg = AllocateCpuRegister(); |
| const auto& container_type = ConvertFloatToInt(zone_, payload_type); |
| return *new (zone_) |
| NativeRegistersLocation(zone_, payload_type, container_type, reg); |
| } |
| #elif defined(TARGET_ARCH_RISCV32) |
| // After using up F registers, start bitcasting to X register (pairs). |
| if (((payload_type.SizeInBytes() == 4) && HasAvailableCpuRegisters(1)) || |
| ((payload_type.SizeInBytes() == 8) && HasAvailableCpuRegisters(2))) { |
| const auto& container_type = ConvertFloatToInt(zone_, payload_type); |
| return AllocateInt(payload_type, container_type, is_vararg); |
| } |
| #endif |
| |
| BlockAllFpuRegisters(); |
| if (CallingConventions::kArgumentIntRegXorFpuReg) { |
| ASSERT(cpu_regs_used == CallingConventions::kNumArgRegs); |
| } |
| return AllocateStack(payload_type); |
| } |
| |
| const NativeLocation& AllocateInt(const NativeType& payload_type, |
| const NativeType& container_type, |
| bool is_vararg) { |
| if (target::kWordSize == 4 && payload_type.SizeInBytes() == 8) { |
| if (CallingConventions::kArgumentRegisterAlignment == |
| kAlignedToWordSizeAndValueSize || |
| (is_vararg && CallingConventions::kArgumentRegisterAlignmentVarArgs == |
| kAlignedToWordSizeAndValueSize)) { |
| cpu_regs_used += cpu_regs_used % 2; |
| } |
| if (cpu_regs_used + 2 <= CallingConventions::kNumArgRegs) { |
| const Register register_1 = AllocateCpuRegister(); |
| const Register register_2 = AllocateCpuRegister(); |
| return *new (zone_) NativeRegistersLocation( |
| zone_, payload_type, container_type, register_1, register_2); |
| } |
| } else { |
| ASSERT(payload_type.SizeInBytes() <= target::kWordSize); |
| if (cpu_regs_used + 1 <= CallingConventions::kNumArgRegs) { |
| return *new (zone_) NativeRegistersLocation( |
| zone_, payload_type, container_type, AllocateCpuRegister()); |
| } |
| } |
| return AllocateStack(payload_type, is_vararg); |
| } |
| |
| // Constructs a container type. |
| const NativeLocation& AllocateInt(const NativeType& payload_type, |
| bool is_vararg) { |
| const auto& payload_type_converted = |
| ConvertIfSoftFp(zone_, payload_type, has_varargs_); |
| |
| // Some calling conventions require the callee to make the lowest 32 bits |
| // in registers non-garbage. |
| const auto& container_type = payload_type_converted.Extend( |
| zone_, CallingConventions::kArgumentRegisterExtension); |
| |
| return AllocateInt(payload_type, container_type, is_vararg); |
| } |
| |
| #if defined(TARGET_ARCH_X64) && !defined(DART_TARGET_OS_WINDOWS) |
| // If fits in two fpu and/or cpu registers, transfer in those. Otherwise, |
| // transfer on stack. |
| const NativeLocation& AllocateCompound(const NativeCompoundType& payload_type, |
| bool is_vararg, |
| bool is_result) { |
| const intptr_t size = payload_type.SizeInBytes(); |
| if (size <= 16 && size > 0 && !payload_type.ContainsUnalignedMembers()) { |
| intptr_t required_regs = |
| payload_type.NumberOfWordSizeChunksNotOnlyFloat(); |
| intptr_t required_xmm_regs = |
| payload_type.NumberOfWordSizeChunksOnlyFloat(); |
| const bool regs_available = |
| cpu_regs_used + required_regs <= CallingConventions::kNumArgRegs; |
| const bool fpu_regs_available = |
| FirstFreeFpuRegisterIndex(kQuadFpuReg) != kNoFpuRegister && |
| FirstFreeFpuRegisterIndex(kQuadFpuReg) + required_xmm_regs <= |
| CallingConventions::kNumFpuArgRegs; |
| if (regs_available && fpu_regs_available) { |
| // Transfer in registers. |
| NativeLocations& multiple_locations = *new (zone_) NativeLocations( |
| zone_, required_regs + required_xmm_regs); |
| for (intptr_t offset = 0; offset < size; |
| offset += compiler::target::kWordSize) { |
| if (payload_type.ContainsOnlyFloats(Range::StartAndEnd( |
| offset, Utils::Minimum<intptr_t>(size, offset + 8)))) { |
| const intptr_t reg_index = FirstFreeFpuRegisterIndex(kQuadFpuReg); |
| AllocateFpuRegisterAtIndex(kQuadFpuReg, reg_index); |
| const auto& type = *new (zone_) NativePrimitiveType(kDouble); |
| multiple_locations.Add(new (zone_) NativeFpuRegistersLocation( |
| type, type, kQuadFpuReg, reg_index)); |
| } else { |
| const auto& payload_type = |
| *new (zone_) NativePrimitiveType(TypeForSize(Utils::Minimum( |
| size - offset, compiler::target::kWordSize))); |
| const auto& container_type = *new (zone_) NativePrimitiveType( |
| TypeForSize(compiler::target::kWordSize)); |
| multiple_locations.Add(new (zone_) NativeRegistersLocation( |
| zone_, payload_type, container_type, AllocateCpuRegister())); |
| } |
| } |
| return *new (zone_) |
| MultipleNativeLocations(payload_type, multiple_locations); |
| } |
| } |
| return AllocateStack(payload_type); |
| } |
| #endif // defined(TARGET_ARCH_X64) && !defined(DART_TARGET_OS_WINDOWS) |
| |
| #if defined(TARGET_ARCH_X64) && defined(DART_TARGET_OS_WINDOWS) |
| // If struct fits in a single register and size is a power of two, then |
| // use a single register and sign extend. |
| // Otherwise, pass a pointer to a copy. |
| const NativeLocation& AllocateCompound(const NativeCompoundType& payload_type, |
| bool is_vararg, |
| bool is_result) { |
| const NativeCompoundType& compound_type = payload_type.AsCompound(); |
| const intptr_t size = compound_type.SizeInBytes(); |
| if (size <= 8 && Utils::IsPowerOfTwo(size)) { |
| if (cpu_regs_used < CallingConventions::kNumArgRegs) { |
| NativeLocations& multiple_locations = |
| *new (zone_) NativeLocations(zone_, 1); |
| const auto& type = *new (zone_) NativePrimitiveType( |
| PrimitiveTypeFromSizeInBytes(size)); |
| multiple_locations.Add(new (zone_) NativeRegistersLocation( |
| zone_, type, type, AllocateCpuRegister())); |
| return *new (zone_) |
| MultipleNativeLocations(compound_type, multiple_locations); |
| } |
| |
| } else if (size > 0) { |
| // Pointer in register if available, else pointer on stack. |
| const auto& pointer_type = *new (zone_) NativePrimitiveType(kAddress); |
| const auto& pointer_location = AllocateArgument(pointer_type); |
| return *new (zone_) |
| PointerToMemoryLocation(pointer_location, compound_type); |
| } |
| |
| return AllocateStack(payload_type); |
| } |
| #endif // defined(TARGET_ARCH_X64) && defined(DART_TARGET_OS_WINDOWS) |
| |
| #if defined(TARGET_ARCH_IA32) |
| const NativeLocation& AllocateCompound(const NativeCompoundType& payload_type, |
| bool is_vararg, |
| bool is_result) { |
| return AllocateStack(payload_type); |
| } |
| #endif // defined(TARGET_ARCH_IA32) |
| |
| #if defined(TARGET_ARCH_ARM) |
| // Transfer homogeneous floats in FPU registers, and allocate the rest |
| // in 4 or 8 size chunks in registers and stack. |
| const NativeLocation& AllocateCompound(const NativeCompoundType& payload_type, |
| bool is_vararg, |
| bool is_result) { |
| const auto& compound_type = payload_type.AsCompound(); |
| if (compound_type.ContainsHomogeneousFloats() && |
| !SoftFpAbi(has_varargs_, is_result) && |
| compound_type.NumPrimitiveMembersRecursive() <= 4) { |
| const auto& elem_type = compound_type.FirstPrimitiveMember(); |
| const intptr_t size = compound_type.SizeInBytes(); |
| const intptr_t elem_size = elem_type.SizeInBytes(); |
| const auto reg_kind = FpuRegisterKindFromSize(elem_size); |
| ASSERT(size % elem_size == 0); |
| const intptr_t num_registers = size / elem_size; |
| const intptr_t first_reg = |
| FirstFreeFpuRegisterIndex(reg_kind, num_registers); |
| if (first_reg != kNoFpuRegister) { |
| AllocateFpuRegisterAtIndex(reg_kind, first_reg, num_registers); |
| |
| NativeLocations& multiple_locations = |
| *new (zone_) NativeLocations(zone_, num_registers); |
| for (int i = 0; i < num_registers; i++) { |
| const intptr_t reg_index = first_reg + i; |
| multiple_locations.Add(new (zone_) NativeFpuRegistersLocation( |
| elem_type, elem_type, reg_kind, reg_index)); |
| } |
| return *new (zone_) |
| MultipleNativeLocations(compound_type, multiple_locations); |
| |
| } else { |
| BlockAllFpuRegisters(); |
| return AllocateStack(payload_type); |
| } |
| } else if (payload_type.AlignmentInBytesStack() == 8) { |
| const intptr_t chunk_size = payload_type.AlignmentInBytesStack(); |
| ASSERT(chunk_size == 4 || chunk_size == 8); |
| const intptr_t size_rounded = |
| Utils::RoundUp(payload_type.SizeInBytes(), chunk_size); |
| const intptr_t num_chunks = size_rounded / chunk_size; |
| const auto& chuck_type = |
| *new (zone_) NativePrimitiveType(chunk_size == 4 ? kInt32 : kInt64); |
| |
| NativeLocations& multiple_locations = |
| *new (zone_) NativeLocations(zone_, num_chunks); |
| for (int i = 0; i < num_chunks; i++) { |
| const auto& allocated_chunk = &AllocateArgument(chuck_type); |
| // The last chunk should not be 8 bytes, if the struct only has 4 |
| // remaining bytes to be allocated. |
| if (i == num_chunks - 1 && chunk_size == 8 && |
| Utils::RoundUp(payload_type.SizeInBytes(), 4) % 8 == 4) { |
| const auto& small_chuck_type = *new (zone_) NativePrimitiveType( |
| chunk_size == 4 ? kInt32 : kInt64); |
| multiple_locations.Add(&allocated_chunk->WithOtherNativeType( |
| zone_, small_chuck_type, small_chuck_type)); |
| } else { |
| multiple_locations.Add(allocated_chunk); |
| } |
| } |
| return *new (zone_) |
| MultipleNativeLocations(compound_type, multiple_locations); |
| } else { |
| return AllocateCompoundAsMultiple(compound_type); |
| } |
| } |
| #endif // defined(TARGET_ARCH_ARM) |
| |
| #if defined(TARGET_ARCH_ARM64) |
| // Slightly different from Arm32. FPU registers don't alias the same way, |
| // structs up to 16 bytes block remaining registers if they do not fit in |
| // registers, and larger structs go on stack always. |
| const NativeLocation& AllocateCompound(const NativeCompoundType& payload_type, |
| bool is_vararg, |
| bool is_result) { |
| const auto& compound_type = payload_type.AsCompound(); |
| const intptr_t size = compound_type.SizeInBytes(); |
| if (compound_type.ContainsHomogeneousFloats() && |
| !SoftFpAbi(has_varargs_, is_result) && |
| compound_type.NumPrimitiveMembersRecursive() <= 4) { |
| const auto& elem_type = compound_type.FirstPrimitiveMember(); |
| const intptr_t elem_size = elem_type.SizeInBytes(); |
| const auto reg_kind = kQuadFpuReg; |
| ASSERT(size % elem_size == 0); |
| const intptr_t num_registers = size / elem_size; |
| const intptr_t first_reg = |
| FirstFreeFpuRegisterIndex(reg_kind, num_registers); |
| if (first_reg != kNoFpuRegister) { |
| AllocateFpuRegisterAtIndex(reg_kind, first_reg, num_registers); |
| |
| NativeLocations& multiple_locations = |
| *new (zone_) NativeLocations(zone_, num_registers); |
| for (int i = 0; i < num_registers; i++) { |
| const intptr_t reg_index = first_reg + i; |
| multiple_locations.Add(new (zone_) NativeFpuRegistersLocation( |
| elem_type, elem_type, reg_kind, reg_index)); |
| } |
| return *new (zone_) |
| MultipleNativeLocations(compound_type, multiple_locations); |
| } |
| BlockAllFpuRegisters(); |
| return AllocateStack(payload_type, is_vararg); |
| } |
| |
| if (size <= 16) { |
| const intptr_t size_rounded = Utils::RoundUp(size, 8); |
| const intptr_t num_chunks = size_rounded / 8; |
| ASSERT((num_chunks == 1) || (num_chunks == 2)); |
| |
| // All-or-none: block any leftover registers. |
| #if defined(DART_TARGET_OS_WINDOWS) |
| if (!HasAvailableCpuRegisters(num_chunks) && !is_vararg) { |
| cpu_regs_used = CallingConventions::kNumArgRegs; |
| } |
| #else |
| if (!HasAvailableCpuRegisters(num_chunks)) { |
| cpu_regs_used = CallingConventions::kNumArgRegs; |
| } |
| #endif |
| |
| return AllocateCompoundAsMultiple(payload_type); |
| } |
| |
| const auto& pointer_location = |
| AllocateArgument(*new (zone_) NativePrimitiveType(kInt64)); |
| return *new (zone_) |
| PointerToMemoryLocation(pointer_location, compound_type); |
| } |
| #endif // defined(TARGET_ARCH_ARM64) |
| |
| #if defined(TARGET_ARCH_RISCV32) || defined(TARGET_ARCH_RISCV64) |
| // See RISC-V ABIs Specification |
| // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/releases |
| const NativeLocation& AllocateCompound(const NativeCompoundType& payload_type, |
| bool is_vararg, |
| bool is_result) { |
| const auto& compound_type = payload_type.AsCompound(); |
| |
| // 2.2. Hardware Floating-point Calling Convention. |
| const NativePrimitiveType* first = nullptr; |
| const NativePrimitiveType* second = nullptr; |
| const intptr_t num_primitive_members = |
| compound_type.PrimitivePairMembers(&first, &second); |
| |
| // If exactly one floating-point member, pass like a scalar. |
| if ((num_primitive_members == 1) && first->IsFloat()) { |
| NativeLocations& multiple_locations = |
| *new (zone_) NativeLocations(zone_, 1); |
| multiple_locations.Add(&AllocateArgument(*first)); |
| return *new (zone_) |
| MultipleNativeLocations(compound_type, multiple_locations); |
| } |
| |
| if (num_primitive_members == 2) { |
| if (first->IsFloat() && second->IsFloat()) { |
| // If exactly two floating-point members, pass like two scalars if two F |
| // registers are available. |
| if (HasAvailableFpuRegisters(2)) { |
| NativeLocations& multiple_locations = |
| *new (zone_) NativeLocations(zone_, 2); |
| multiple_locations.Add(&AllocateArgument(*first)); |
| multiple_locations.Add(&AllocateArgument(*second)); |
| return *new (zone_) |
| MultipleNativeLocations(compound_type, multiple_locations); |
| } |
| } else if (first->IsFloat() || second->IsFloat()) { |
| // If exactly two members, one is integer and one is float in either |
| // order, pass like two scalars if both an X and F register are |
| // available. |
| if (HasAvailableFpuRegisters(1) && HasAvailableCpuRegisters(1)) { |
| NativeLocations& multiple_locations = |
| *new (zone_) NativeLocations(zone_, 2); |
| multiple_locations.Add(&AllocateArgument(*first)); |
| multiple_locations.Add(&AllocateArgument(*second)); |
| return *new (zone_) |
| MultipleNativeLocations(compound_type, multiple_locations); |
| } |
| } |
| } |
| |
| // 2.1. Integer Calling Convention. |
| // If total size is <= XLEN, passed like an XLEN scalar: use a register if |
| // available or pass by value on the stack. |
| // If total size is <= 2*XLEN, passed like two XLEN scalars: use registers |
| // if available or pass by value on the stack. If only one register is |
| // available, pass the low part by register and the high part on the |
| // stack. |
| if (compound_type.SizeInBytes() <= 2 * target::kWordSize) { |
| return AllocateCompoundAsMultiple(compound_type); |
| } |
| |
| // Otherwise, passed by reference. |
| const auto& pointer_type = *new (zone_) NativePrimitiveType(kAddress); |
| const auto& pointer_location = AllocateArgument(pointer_type); |
| return *new (zone_) |
| PointerToMemoryLocation(pointer_location, compound_type); |
| } |
| #endif |
| |
| // Allocate in word-sized chunks, with the container as a full word-sized |
| // register or stack slot and the payload constrained to the struct's size. |
| // |
| // Note this describes the location at call. Consumes of this location, such |
| // as FfiCallConvertCompoundArgumentToNative or EmitReturnMoves, often assume |
| // the location of the source compound in the heap corresponds to this |
| // location with just a change in base register. This is often true, except |
| // some ABIs assume zero extension of the last chunk, so the stack location at |
| // call is bigger than the location in the heap. Here we set the container |
| // size to reflect that zero-extended stack slot and rely on loads during |
| // moves opting to use the payload size instead of the container size to stay |
| // in-bounds. |
| const NativeLocation& AllocateCompoundAsMultiple( |
| const NativeCompoundType& compound_type) { |
| const intptr_t chunk_size = compiler::target::kWordSize; |
| const intptr_t num_chunks = |
| Utils::RoundUp(compound_type.SizeInBytes(), chunk_size) / chunk_size; |
| const NativeType& container_type = |
| *new (zone_) NativePrimitiveType(TypeForSize(chunk_size)); |
| NativeLocations& locations = |
| *new (zone_) NativeLocations(zone_, num_chunks); |
| intptr_t size_remaining = compound_type.SizeInBytes(); |
| while (size_remaining > 0) { |
| const auto& chunk = AllocateArgument(container_type); |
| |
| const intptr_t size = Utils::Minimum(size_remaining, chunk_size); |
| const NativeType& payload_type = |
| *new (zone_) NativePrimitiveType(TypeForSize(size)); |
| locations.Add( |
| &chunk.WithOtherNativeType(zone_, payload_type, container_type)); |
| size_remaining -= size; |
| } |
| return *new (zone_) MultipleNativeLocations(compound_type, locations); |
| } |
| |
| static FpuRegisterKind FpuRegKind(const NativeType& payload_type) { |
| #if defined(TARGET_ARCH_ARM) |
| return FpuRegisterKindFromSize(payload_type.SizeInBytes()); |
| #else |
| return kQuadFpuReg; |
| #endif |
| } |
| |
| Register AllocateCpuRegister() { |
| RELEASE_ASSERT(cpu_regs_used >= 0); // Avoids -Werror=array-bounds in GCC. |
| ASSERT(cpu_regs_used < CallingConventions::kNumArgRegs); |
| |
| const auto result = CallingConventions::ArgumentRegisters[cpu_regs_used]; |
| if (CallingConventions::kArgumentIntRegXorFpuReg) { |
| AllocateFpuRegisterAtIndex(kQuadFpuReg, cpu_regs_used); |
| } |
| cpu_regs_used++; |
| return result; |
| } |
| |
| const NativeLocation& AllocateStack(const NativeType& payload_type, |
| bool is_vararg = false) { |
| align_stack(payload_type.AlignmentInBytesStack(is_vararg)); |
| const intptr_t size = payload_type.SizeInBytes(); |
| // If the stack arguments are not packed, the 32 lowest bits should not |
| // contain garbage. |
| const auto& container_type = |
| payload_type.Extend(zone_, CallingConventions::kArgumentStackExtension); |
| const auto& result = *new (zone_) NativeStackLocation( |
| payload_type, container_type, CallingConventions::kStackPointerRegister, |
| stack_height_in_bytes); |
| stack_height_in_bytes += size; |
| align_stack(payload_type.AlignmentInBytesStack(is_vararg)); |
| return result; |
| } |
| |
| void align_stack(intptr_t alignment) { |
| stack_height_in_bytes = Utils::RoundUp(stack_height_in_bytes, alignment); |
| } |
| |
| int NumFpuRegisters(FpuRegisterKind kind) const { |
| #if defined(TARGET_ARCH_ARM) |
| if (has_varargs_) return 0; |
| if (kind == kSingleFpuReg) return CallingConventions::kNumSFpuArgRegs; |
| if (kind == kDoubleFpuReg) return CallingConventions::kNumDFpuArgRegs; |
| #endif // defined(TARGET_ARCH_ARM) |
| if (kind == kQuadFpuReg) return CallingConventions::kNumFpuArgRegs; |
| UNREACHABLE(); |
| } |
| |
| // If no register is free, returns -1. |
| int FirstFreeFpuRegisterIndex(FpuRegisterKind kind, int amount = 1) const { |
| const intptr_t size = SizeFromFpuRegisterKind(kind) / 4; |
| ASSERT(size == 1 || size == 2 || size == 4); |
| if (fpu_reg_parts_used == -1) return kNoFpuRegister; |
| const intptr_t mask = (1 << (size * amount)) - 1; |
| intptr_t index = 0; |
| while (index + amount <= NumFpuRegisters(kind)) { |
| const intptr_t mask_shifted = mask << (index * size); |
| if ((fpu_reg_parts_used & mask_shifted) == 0) { |
| return index; |
| } |
| index++; |
| } |
| return kNoFpuRegister; |
| } |
| |
| void AllocateFpuRegisterAtIndex(FpuRegisterKind kind, |
| int index, |
| int amount = 1) { |
| const intptr_t size = SizeFromFpuRegisterKind(kind) / 4; |
| ASSERT(size == 1 || size == 2 || size == 4); |
| const intptr_t mask = (1 << size * amount) - 1; |
| const intptr_t mask_shifted = (mask << (index * size)); |
| ASSERT((mask_shifted & fpu_reg_parts_used) == 0); |
| fpu_reg_parts_used |= mask_shifted; |
| } |
| |
| // > The back-filling continues only so long as no VFP CPRC has been |
| // > allocated to a slot on the stack. |
| // Procedure Call Standard for the Arm Architecture, Release 2019Q1.1 |
| // Chapter 7.1 page 28. https://developer.arm.com/docs/ihi0042/h |
| // |
| // Irrelevant on Android and iOS, as those are both SoftFP. |
| // > For floating-point arguments, the Base Standard variant of the |
| // > Procedure Call Standard is used. In this variant, floating-point |
| // > (and vector) arguments are passed in general purpose registers |
| // > (GPRs) instead of in VFP registers) |
| // https://developer.apple.com/library/archive/documentation/Xcode/Conceptual/iPhoneOSABIReference/Articles/ARMv7FunctionCallingConventions.html#//apple_ref/doc/uid/TP40009022-SW1 |
| void BlockAllFpuRegisters() { |
| // Set all bits to 1. |
| fpu_reg_parts_used = -1; |
| } |
| |
| bool HasAvailableCpuRegisters(intptr_t count) const { |
| return cpu_regs_used + count <= CallingConventions::kNumArgRegs; |
| } |
| bool HasAvailableFpuRegisters(intptr_t count) const { |
| return FirstFreeFpuRegisterIndex(kQuadFpuReg, count) != kNoFpuRegister; |
| } |
| |
| intptr_t cpu_regs_used = 0; |
| // Every bit denotes 32 bits of FPU registers. |
| intptr_t fpu_reg_parts_used = 0; |
| intptr_t stack_height_in_bytes = 0; |
| const bool has_varargs_; |
| Zone* zone_; |
| }; |
| |
| // Location for the arguments of a C signature function. |
| static NativeLocations& ArgumentLocations( |
| Zone* zone, |
| const ZoneGrowableArray<const NativeType*>& arg_reps, |
| const NativeLocation& return_location, |
| intptr_t var_args_index) { |
| intptr_t num_arguments = arg_reps.length(); |
| auto& result = *new (zone) NativeLocations(zone, num_arguments); |
| |
| // Loop through all arguments and assign a register or a stack location. |
| // Allocate result pointer for composite returns first. |
| const bool has_varargs = |
| var_args_index != NativeFunctionType::kNoVariadicArguments; |
| ArgumentAllocator frame_state(zone, has_varargs); |
| #if !defined(TARGET_ARCH_ARM64) |
| // Arm64 allocates the pointer in R8, which is not an argument location. |
| if (return_location.IsPointerToMemory()) { |
| const auto& pointer_location = |
| return_location.AsPointerToMemory().pointer_location(); |
| const auto& pointer_location_allocated = |
| frame_state.AllocateArgumentVariadic(pointer_location.payload_type()); |
| ASSERT(pointer_location.Equals(pointer_location_allocated)); |
| } |
| #endif |
| |
| for (intptr_t i = 0; i < num_arguments; i++) { |
| const NativeType& rep = *arg_reps[i]; |
| const bool is_first_vararg = has_varargs && i == var_args_index; |
| const bool is_vararg = has_varargs && i >= var_args_index; |
| result.Add( |
| &frame_state.AllocateArgumentVariadic(rep, is_first_vararg, is_vararg)); |
| } |
| return result; |
| } |
| |
| #if !defined(TARGET_ARCH_IA32) |
| static const NativeLocation& PointerToMemoryResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type) { |
| const auto& pointer_type = *new (zone) NativePrimitiveType(kAddress); |
| const auto& pointer_location = *new (zone) NativeRegistersLocation( |
| zone, pointer_type, pointer_type, |
| CallingConventions::kPointerToReturnStructRegisterCall); |
| const auto& pointer_return_location = *new (zone) NativeRegistersLocation( |
| zone, pointer_type, pointer_type, |
| CallingConventions::kPointerToReturnStructRegisterReturn); |
| return *new (zone) PointerToMemoryLocation( |
| pointer_location, pointer_return_location, payload_type); |
| } |
| #endif // !defined(TARGET_ARCH_IA32) |
| |
| #if defined(TARGET_ARCH_IA32) |
| // ia32 Passes pointers to result locations on the stack. |
| static const NativeLocation& PointerToMemoryResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type) { |
| const auto& pointer_type = *new (zone) NativePrimitiveType(kAddress); |
| const auto& pointer_location = *new (zone) NativeStackLocation( |
| pointer_type, pointer_type, CallingConventions::kStackPointerRegister, 0); |
| const auto& pointer_return_location = *new (zone) NativeRegistersLocation( |
| zone, pointer_type, pointer_type, |
| CallingConventions::kPointerToReturnStructRegisterReturn); |
| return *new (zone) PointerToMemoryLocation( |
| pointer_location, pointer_return_location, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_IA32) |
| |
| #if defined(TARGET_ARCH_X64) && !defined(DART_TARGET_OS_WINDOWS) |
| static const NativeLocation& CompoundResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type, |
| bool has_varargs) { |
| const intptr_t size = payload_type.SizeInBytes(); |
| if (size <= 16 && size > 0 && !payload_type.ContainsUnalignedMembers()) { |
| // Allocate the same as argument, but use return registers instead of |
| // argument registers. |
| NativeLocations& multiple_locations = |
| *new (zone) NativeLocations(zone, size > 8 ? 2 : 1); |
| intptr_t used_regs = 0; |
| intptr_t used_xmm_regs = 0; |
| |
| const auto& double_type = *new (zone) NativePrimitiveType(kDouble); |
| |
| const bool first_half_in_xmm = payload_type.ContainsOnlyFloats( |
| Range::StartAndEnd(0, Utils::Minimum<intptr_t>(size, 8))); |
| if (first_half_in_xmm) { |
| multiple_locations.Add(new (zone) NativeFpuRegistersLocation( |
| double_type, double_type, kQuadFpuReg, |
| CallingConventions::kReturnFpuReg)); |
| used_xmm_regs++; |
| } else { |
| const auto& payload_type = *new (zone) NativePrimitiveType( |
| TypeForSize(Utils::Minimum(size, compiler::target::kWordSize))); |
| const auto& container_type = *new (zone) NativePrimitiveType( |
| TypeForSize(compiler::target::kWordSize)); |
| multiple_locations.Add(new (zone) NativeRegistersLocation( |
| zone, payload_type, container_type, CallingConventions::kReturnReg)); |
| used_regs++; |
| } |
| if (size > 8) { |
| const bool second_half_in_xmm = payload_type.ContainsOnlyFloats( |
| Range::StartAndEnd(8, Utils::Minimum<intptr_t>(size, 16))); |
| if (second_half_in_xmm) { |
| const FpuRegister reg = used_xmm_regs == 0 |
| ? CallingConventions::kReturnFpuReg |
| : CallingConventions::kSecondReturnFpuReg; |
| multiple_locations.Add(new (zone) NativeFpuRegistersLocation( |
| double_type, double_type, kQuadFpuReg, reg)); |
| used_xmm_regs++; |
| } else { |
| const Register reg = used_regs == 0 |
| ? CallingConventions::kReturnReg |
| : CallingConventions::kSecondReturnReg; |
| const auto& payload_type = *new (zone) NativePrimitiveType( |
| TypeForSize(Utils::Minimum(size - 8, compiler::target::kWordSize))); |
| const auto& container_type = *new (zone) NativePrimitiveType( |
| TypeForSize(compiler::target::kWordSize)); |
| multiple_locations.Add(new (zone) NativeRegistersLocation( |
| zone, payload_type, container_type, reg)); |
| used_regs++; |
| } |
| } |
| return *new (zone) |
| MultipleNativeLocations(payload_type, multiple_locations); |
| } |
| return PointerToMemoryResultLocation(zone, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_X64) && !defined(DART_TARGET_OS_WINDOWS) |
| |
| #if defined(TARGET_ARCH_X64) && defined(DART_TARGET_OS_WINDOWS) |
| // If struct fits in a single register do that, and sign extend. |
| // Otherwise, pass a pointer to memory. |
| static const NativeLocation& CompoundResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type, |
| bool has_varargs) { |
| const intptr_t size = payload_type.SizeInBytes(); |
| if (size <= 8 && size > 0 && Utils::IsPowerOfTwo(size)) { |
| NativeLocations& multiple_locations = *new (zone) NativeLocations(zone, 1); |
| const auto& type = |
| *new (zone) NativePrimitiveType(PrimitiveTypeFromSizeInBytes(size)); |
| multiple_locations.Add(new (zone) NativeRegistersLocation( |
| zone, type, type, CallingConventions::kReturnReg)); |
| return *new (zone) |
| MultipleNativeLocations(payload_type, multiple_locations); |
| } |
| return PointerToMemoryResultLocation(zone, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_X64) && defined(DART_TARGET_OS_WINDOWS) |
| |
| #if defined(TARGET_ARCH_IA32) && !defined(DART_TARGET_OS_WINDOWS) |
| static const NativeLocation& CompoundResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type, |
| bool has_varargs) { |
| return PointerToMemoryResultLocation(zone, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_IA32) && !defined(DART_TARGET_OS_WINDOWS) |
| |
| #if defined(TARGET_ARCH_IA32) && defined(DART_TARGET_OS_WINDOWS) |
| // Windows uses up to two return registers, while Linux does not. |
| static const NativeLocation& CompoundResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type, |
| bool has_varargs) { |
| const intptr_t size = payload_type.SizeInBytes(); |
| if (size <= 8 && Utils::IsPowerOfTwo(size)) { |
| NativeLocations& multiple_locations = |
| *new (zone) NativeLocations(zone, size > 4 ? 2 : 1); |
| const auto& type = *new (zone) NativePrimitiveType(kUint32); |
| multiple_locations.Add(new (zone) NativeRegistersLocation( |
| zone, type, type, CallingConventions::kReturnReg)); |
| if (size > 4) { |
| multiple_locations.Add(new (zone) NativeRegistersLocation( |
| zone, type, type, CallingConventions::kSecondReturnReg)); |
| } |
| return *new (zone) |
| MultipleNativeLocations(payload_type, multiple_locations); |
| } |
| return PointerToMemoryResultLocation(zone, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_IA32) && defined(DART_TARGET_OS_WINDOWS) |
| |
| #if defined(TARGET_ARCH_ARM) |
| // Arm passes homogenous float return values in FPU registers and small |
| // composites in a single integer register. The rest is stored into the |
| // location passed in by pointer. |
| static const NativeLocation& CompoundResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type, |
| bool has_varargs) { |
| const intptr_t num_members = payload_type.NumPrimitiveMembersRecursive(); |
| if (payload_type.ContainsHomogeneousFloats() && |
| !SoftFpAbi(has_varargs, /*is_result*/ true) && num_members <= 4) { |
| NativeLocations& multiple_locations = |
| *new (zone) NativeLocations(zone, num_members); |
| for (int i = 0; i < num_members; i++) { |
| const auto& member = payload_type.FirstPrimitiveMember(); |
| multiple_locations.Add(new (zone) NativeFpuRegistersLocation( |
| member, member, FpuRegisterKindFromSize(member.SizeInBytes()), i)); |
| } |
| return *new (zone) |
| MultipleNativeLocations(payload_type, multiple_locations); |
| } |
| const intptr_t size = payload_type.SizeInBytes(); |
| if (size <= 4) { |
| NativeLocations& multiple_locations = *new (zone) NativeLocations(zone, 1); |
| const auto& type = *new (zone) NativePrimitiveType(kUint32); |
| multiple_locations.Add(new (zone) |
| NativeRegistersLocation(zone, type, type, R0)); |
| return *new (zone) |
| MultipleNativeLocations(payload_type, multiple_locations); |
| } |
| return PointerToMemoryResultLocation(zone, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_ARM) |
| |
| #if defined(TARGET_ARCH_ARM64) |
| // If allocated to integer or fpu registers as argument, same for return, |
| // otherwise a pointer to the result location is passed in. |
| static const NativeLocation& CompoundResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type, |
| bool has_varargs) { |
| ArgumentAllocator frame_state(zone, has_varargs); |
| const auto& location_as_argument = |
| frame_state.AllocateArgumentVariadic(payload_type); |
| if (!location_as_argument.IsStack() && |
| !location_as_argument.IsPointerToMemory()) { |
| return location_as_argument; |
| } |
| return PointerToMemoryResultLocation(zone, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_ARM64) |
| |
| #if defined(TARGET_ARCH_RISCV32) || defined(TARGET_ARCH_RISCV64) |
| static const NativeLocation& CompoundResultLocation( |
| Zone* zone, |
| const NativeCompoundType& payload_type, |
| bool has_varargs) { |
| // First or first and second argument registers if it fits, otherwise a |
| // pointer to the result location is passed in. |
| ArgumentAllocator frame_state(zone, has_varargs); |
| const auto& location_as_argument = |
| frame_state.AllocateArgumentVariadic(payload_type); |
| if (!location_as_argument.IsStack() && |
| !location_as_argument.IsPointerToMemory()) { |
| return location_as_argument; |
| } |
| return PointerToMemoryResultLocation(zone, payload_type); |
| } |
| #endif // defined(TARGET_ARCH_RISCV32) || defined(TARGET_ARCH_RISCV64) |
| |
| // Location for the result of a C signature function. |
| static const NativeLocation& ResultLocation(Zone* zone, |
| const NativeType& payload_type, |
| bool has_varargs) { |
| const auto& payload_type_converted = |
| ConvertIfSoftFp(zone, payload_type, has_varargs, /*is_result*/ true); |
| const auto& container_type = payload_type_converted.Extend( |
| zone, CallingConventions::kReturnRegisterExtension); |
| |
| if (container_type.IsFloat()) { |
| return *new (zone) NativeFpuRegistersLocation( |
| payload_type, container_type, CallingConventions::kReturnFpuReg); |
| } |
| |
| if (container_type.IsInt() || container_type.IsVoid()) { |
| if (container_type.SizeInBytes() == 8 && target::kWordSize == 4) { |
| return *new (zone) NativeRegistersLocation( |
| zone, payload_type, container_type, CallingConventions::kReturnReg, |
| CallingConventions::kSecondReturnReg); |
| } |
| |
| ASSERT(container_type.SizeInBytes() <= target::kWordSize); |
| return *new (zone) NativeRegistersLocation( |
| zone, payload_type, container_type, CallingConventions::kReturnReg); |
| } |
| |
| // Compounds are laid out differently per ABI, so they are implemented |
| // per ABI. |
| const auto& compound_type = payload_type.AsCompound(); |
| return CompoundResultLocation(zone, compound_type, has_varargs); |
| } |
| |
| const NativeCallingConvention& NativeCallingConvention::FromSignature( |
| Zone* zone, |
| const NativeFunctionType& signature) { |
| const bool contains_varargs = signature.variadic_arguments_index() != |
| NativeFunctionType::kNoVariadicArguments; |
| // With struct return values, a possible pointer to a return value can |
| // occupy an argument position. Hence, allocate return value first. |
| const auto& return_location = |
| ResultLocation(zone, signature.return_type(), contains_varargs); |
| const auto& argument_locations = |
| ArgumentLocations(zone, signature.argument_types(), return_location, |
| signature.variadic_arguments_index()); |
| return *new (zone) NativeCallingConvention(argument_locations, |
| return_location, contains_varargs); |
| } |
| |
| intptr_t NativeCallingConvention::StackTopInBytes() const { |
| const intptr_t num_arguments = argument_locations_.length(); |
| intptr_t max_height_in_bytes = 0; |
| for (intptr_t i = 0; i < num_arguments; i++) { |
| max_height_in_bytes = Utils::Maximum( |
| max_height_in_bytes, argument_locations_[i]->StackTopInBytes()); |
| } |
| if (return_location_.IsPointerToMemory()) { |
| const auto& ret_loc = return_location_.AsPointerToMemory(); |
| max_height_in_bytes = |
| Utils::Maximum(max_height_in_bytes, ret_loc.StackTopInBytes()); |
| } |
| return Utils::RoundUp(max_height_in_bytes, compiler::target::kWordSize); |
| } |
| |
| void NativeCallingConvention::PrintTo(BaseTextBuffer* f, |
| bool multi_line) const { |
| if (!multi_line) { |
| f->AddString("("); |
| } |
| for (intptr_t i = 0; i < argument_locations_.length(); i++) { |
| if (i > 0) { |
| if (multi_line) { |
| f->AddString("\n"); |
| } else { |
| f->AddString(", "); |
| } |
| } |
| argument_locations_[i]->PrintTo(f); |
| } |
| if (multi_line) { |
| f->AddString("\n=>\n"); |
| } else { |
| f->AddString(") => "); |
| } |
| return_location_.PrintTo(f); |
| if (multi_line) { |
| f->AddString("\n"); |
| } |
| } |
| |
| const char* NativeCallingConvention::ToCString(Zone* zone, |
| bool multi_line) const { |
| ZoneTextBuffer textBuffer(zone); |
| PrintTo(&textBuffer, multi_line); |
| return textBuffer.buffer(); |
| } |
| |
| #if !defined(FFI_UNIT_TESTS) |
| const char* NativeCallingConvention::ToCString(bool multi_line) const { |
| return ToCString(Thread::Current()->zone(), multi_line); |
| } |
| #endif |
| |
| } // namespace ffi |
| |
| } // namespace compiler |
| |
| } // namespace dart |
