blob: 93febd1c53406a9b6b1eae93c832ce7a5fe56020 [file] [log] [blame] [edit]
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/ffi_callback_metadata.h"
#include "vm/compiler/assembler/disassembler.h"
#include "vm/dart_api_state.h"
#include "vm/flag_list.h"
#include "vm/object.h"
#include "vm/runtime_entry.h"
#include "vm/stub_code.h"
namespace dart {
FfiCallbackMetadata::FfiCallbackMetadata() {}
void FfiCallbackMetadata::EnsureStubPageLocked() {
ASSERT(lock_.IsOwnedByCurrentThread());
if (stub_page_ != nullptr) {
return;
}
ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize);
const Code& trampoline_code = StubCode::FfiCallbackTrampoline();
const uword code_start = trampoline_code.EntryPoint();
const uword code_end = code_start + trampoline_code.Size();
const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1);
ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(),
RXMappingSize());
// Stub page uses a tight (unaligned) bound for the end of the code area.
// Otherwise we can read past the end of the code area when doing DuplicateRX.
stub_page_ = VirtualMemory::ForImagePage(reinterpret_cast<void*>(page_start),
code_end - page_start);
offset_of_first_trampoline_in_page_ = code_start - page_start;
#if defined(DART_TARGET_OS_FUCHSIA)
// On Fuchsia we can't currently duplicate pages, so use the first page of
// trampolines. Store the stub page's metadata in a separately allocated RW
// page.
// TODO(https://dartbug.com/52579): Remove.
fuchsia_metadata_page_ = VirtualMemory::AllocateAligned(
MappingSize(), MappingAlignment(), /*is_executable=*/false,
/*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage");
Metadata* metadata = reinterpret_cast<Metadata*>(
fuchsia_metadata_page_->start() + MetadataOffset());
for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) {
AddToFreeListLocked(&metadata[i]);
}
#endif // defined(DART_TARGET_OS_FUCHSIA)
}
FfiCallbackMetadata::~FfiCallbackMetadata() {
// Unmap all the trampoline pages. 'VirtualMemory's are new-allocated.
delete stub_page_;
for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) {
delete trampoline_pages_[i];
}
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
delete fuchsia_metadata_page_;
#endif // defined(DART_TARGET_OS_FUCHSIA)
}
void FfiCallbackMetadata::Init() {
ASSERT(singleton_ == nullptr);
singleton_ = new FfiCallbackMetadata();
}
void FfiCallbackMetadata::Cleanup() {
ASSERT(singleton_ != nullptr);
delete singleton_;
singleton_ = nullptr;
}
FfiCallbackMetadata* FfiCallbackMetadata::Instance() {
ASSERT(singleton_ != nullptr);
return singleton_;
}
void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page,
uword index,
void* function) {
void** slot =
reinterpret_cast<void**>(page->start() + RuntimeFunctionOffset(index));
*slot = function;
}
VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() {
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
UNREACHABLE();
return nullptr;
#else
#if defined(DART_HOST_OS_MACOS) && !defined(DART_PRECOMPILED_RUNTIME)
// If we are not going to use vm_remap then we need to pass
// is_executable=true so that pages get allocated with MAP_JIT flag if
// necessary. Otherwise OS will kill us with a codesigning violation if
// hardened runtime is enabled.
const bool is_executable = true;
#else
const bool is_executable = false;
#endif
VirtualMemory* new_page = VirtualMemory::AllocateAligned(
MappingSize(), MappingAlignment(), is_executable,
/*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage");
if (new_page == nullptr) {
return nullptr;
}
if (!stub_page_->DuplicateRX(new_page)) {
delete new_page;
return nullptr;
}
#if !defined(PRODUCT) || defined(FORCE_INCLUDE_DISASSEMBLER)
if (FLAG_support_disassembler && FLAG_disassemble_stubs) {
DisassembleToStdout formatter;
THR_Print("Code for duplicated stub 'FfiCallbackTrampoline' {\n");
const uword code_start =
new_page->start() + offset_of_first_trampoline_in_page_;
Disassembler::Disassemble(code_start, code_start + kPageSize, &formatter,
/*comments=*/nullptr);
THR_Print("}\n");
}
#endif
return new_page;
#endif // defined(DART_TARGET_OS_FUCHSIA)
}
void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() {
ASSERT(lock_.IsOwnedByCurrentThread());
EnsureStubPageLocked();
if (free_list_head_ != nullptr) {
return;
}
VirtualMemory* new_page = AllocateTrampolinePage();
if (new_page == nullptr) {
Exceptions::ThrowOOM();
}
trampoline_pages_.Add(new_page);
// Fill in the runtime functions.
FillRuntimeFunction(new_page, kGetFfiCallbackMetadata,
reinterpret_cast<void*>(DLRT_GetFfiCallbackMetadata));
FillRuntimeFunction(new_page, kExitTemporaryIsolate,
reinterpret_cast<void*>(DLRT_ExitTemporaryIsolate));
// Add all the trampolines to the free list.
const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage();
Metadata* metadata =
reinterpret_cast<Metadata*>(new_page->start() + MetadataOffset());
for (intptr_t i = 0; i < trampolines_per_page; ++i) {
AddToFreeListLocked(&metadata[i]);
}
}
FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateMetadataEntry(
Isolate* target_isolate,
TrampolineType trampoline_type,
uword target_entry_point,
uint64_t context,
Metadata** list_head) {
MutexLocker locker(&lock_);
EnsureFreeListNotEmptyLocked();
ASSERT(free_list_head_ != nullptr);
Metadata* entry = free_list_head_;
free_list_head_ = entry->free_list_next_;
if (free_list_head_ == nullptr) {
ASSERT(free_list_tail_ == entry);
free_list_tail_ = nullptr;
}
Metadata* next_entry = *list_head;
if (next_entry != nullptr) {
ASSERT(next_entry->list_prev_ == nullptr);
next_entry->list_prev_ = entry;
}
*entry = Metadata(target_isolate, trampoline_type, target_entry_point,
context, nullptr, next_entry);
*list_head = entry;
return TrampolineOfMetadata(entry);
}
void FfiCallbackMetadata::AddToFreeListLocked(Metadata* entry) {
ASSERT(lock_.IsOwnedByCurrentThread());
if (free_list_tail_ == nullptr) {
ASSERT(free_list_head_ == nullptr);
free_list_head_ = free_list_tail_ = entry;
} else {
ASSERT(free_list_head_ != nullptr && free_list_tail_ != nullptr);
ASSERT(!free_list_tail_->IsLive());
free_list_tail_->free_list_next_ = entry;
free_list_tail_ = entry;
}
entry->context_ = 0;
entry->target_isolate_ = nullptr;
entry->free_list_next_ = nullptr;
}
void FfiCallbackMetadata::DeleteCallbackLocked(Metadata* entry) {
ASSERT(lock_.IsOwnedByCurrentThread());
if (entry->trampoline_type_ != TrampolineType::kAsync &&
entry->context_ != 0) {
ASSERT(entry->target_isolate_ != nullptr);
auto* api_state = entry->target_isolate_->group()->api_state();
ASSERT(api_state != nullptr);
api_state->FreePersistentHandle(entry->closure_handle());
}
AddToFreeListLocked(entry);
}
void FfiCallbackMetadata::DeleteAllCallbacks(Metadata** list_head) {
MutexLocker locker(&lock_);
for (Metadata* entry = *list_head; entry != nullptr;) {
Metadata* next = entry->list_next();
DeleteCallbackLocked(entry);
entry = next;
}
*list_head = nullptr;
}
void FfiCallbackMetadata::DeleteCallback(Trampoline trampoline,
Metadata** list_head) {
MutexLocker locker(&lock_);
auto* entry = MetadataOfTrampoline(trampoline);
ASSERT(entry->IsLive());
auto* prev = entry->list_prev_;
auto* next = entry->list_next_;
if (prev != nullptr) {
prev->list_next_ = next;
} else {
ASSERT(*list_head == entry);
*list_head = next;
}
if (next != nullptr) {
next->list_prev_ = prev;
}
DeleteCallbackLocked(entry);
}
uword FfiCallbackMetadata::GetEntryPoint(Zone* zone, const Function& function) {
const auto& code =
Code::Handle(zone, FLAG_precompiled_mode ? function.CurrentCode()
: function.EnsureHasCode());
ASSERT(!code.IsNull());
return code.EntryPoint();
}
PersistentHandle* FfiCallbackMetadata::CreatePersistentHandle(
Isolate* isolate,
const Closure& closure) {
auto* api_state = isolate->group()->api_state();
ASSERT(api_state != nullptr);
auto* handle = api_state->AllocatePersistentHandle();
handle->set_ptr(closure);
return handle;
}
FfiCallbackMetadata::Trampoline
FfiCallbackMetadata::CreateIsolateLocalFfiCallback(Isolate* isolate,
Zone* zone,
const Function& function,
const Closure& closure,
Metadata** list_head) {
if (closure.IsNull()) {
// If the closure is null, it means the target is a static function, so is
// baked into the trampoline and is an ordinary sync callback.
ASSERT(function.GetFfiCallbackKind() ==
FfiCallbackKind::kIsolateLocalStaticCallback);
return CreateSyncFfiCallbackImpl(isolate, zone, function, nullptr,
list_head);
} else {
ASSERT(function.GetFfiCallbackKind() ==
FfiCallbackKind::kIsolateLocalClosureCallback);
return CreateSyncFfiCallbackImpl(isolate, zone, function,
CreatePersistentHandle(isolate, closure),
list_head);
}
}
FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateSyncFfiCallbackImpl(
Isolate* isolate,
Zone* zone,
const Function& function,
PersistentHandle* closure,
Metadata** list_head) {
TrampolineType trampoline_type = TrampolineType::kSync;
#if defined(TARGET_ARCH_IA32)
// On ia32, store the stack delta that we need to use when returning.
const intptr_t stack_return_delta =
function.FfiCSignatureReturnsStruct() && CallingConventions::kUsesRet4
? compiler::target::kWordSize
: 0;
if (stack_return_delta != 0) {
ASSERT(stack_return_delta == 4);
trampoline_type = TrampolineType::kSyncStackDelta4;
}
#endif
return CreateMetadataEntry(isolate, trampoline_type,
GetEntryPoint(zone, function),
reinterpret_cast<uint64_t>(closure), list_head);
}
FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateAsyncFfiCallback(
Isolate* isolate,
Zone* zone,
const Function& send_function,
Dart_Port send_port,
Metadata** list_head) {
ASSERT(send_function.GetFfiCallbackKind() == FfiCallbackKind::kAsyncCallback);
return CreateMetadataEntry(isolate, TrampolineType::kAsync,
GetEntryPoint(zone, send_function),
static_cast<uint64_t>(send_port), list_head);
}
FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadata(
Metadata* metadata) const {
const uword start = MappingStart(reinterpret_cast<uword>(metadata));
Metadata* metadatas = reinterpret_cast<Metadata*>(start + MetadataOffset());
const uword index = metadata - metadatas;
#if defined(DART_TARGET_OS_FUCHSIA)
return StubCode::FfiCallbackTrampoline().EntryPoint() +
index * kNativeCallbackTrampolineSize;
#else
return start + offset_of_first_trampoline_in_page_ +
index * kNativeCallbackTrampolineSize;
#endif
}
FfiCallbackMetadata::Metadata* FfiCallbackMetadata::MetadataOfTrampoline(
Trampoline trampoline) const {
#if defined(DART_TARGET_OS_FUCHSIA)
// On Fuchsia the metadata page is separate to the trampoline page.
// TODO(https://dartbug.com/52579): Remove.
const uword page_start =
Utils::RoundDown(trampoline - offset_of_first_trampoline_in_page_,
VirtualMemory::PageSize());
const uword index =
(trampoline - offset_of_first_trampoline_in_page_ - page_start) /
kNativeCallbackTrampolineSize;
ASSERT(index < NumCallbackTrampolinesPerPage());
Metadata* metadata_table = reinterpret_cast<Metadata*>(
fuchsia_metadata_page_->start() + MetadataOffset());
return metadata_table + index;
#else
const uword start = MappingStart(trampoline);
Metadata* metadatas = reinterpret_cast<Metadata*>(start + MetadataOffset());
const uword index =
(trampoline - start - offset_of_first_trampoline_in_page_) /
kNativeCallbackTrampolineSize;
return &metadatas[index];
#endif
}
FfiCallbackMetadata::Metadata FfiCallbackMetadata::LookupMetadataForTrampoline(
Trampoline trampoline) const {
return *MetadataOfTrampoline(trampoline);
}
FfiCallbackMetadata* FfiCallbackMetadata::singleton_ = nullptr;
} // namespace dart