runtime/vm/ffi_callback_metadata.cc - sdk.git - Git at Google

 // Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.

 #include "vm/ffi_callback_metadata.h"

 #include "vm/compiler/assembler/disassembler.h"
 #include "vm/dart_api_state.h"
 #include "vm/flag_list.h"
 #include "vm/object.h"
 #include "vm/runtime_entry.h"
 #include "vm/stub_code.h"

 namespace dart {

 FfiCallbackMetadata::FfiCallbackMetadata() {}

 void FfiCallbackMetadata::EnsureStubPageLocked() {
   ASSERT(lock_.IsOwnedByCurrentThread());
   if (stub_page_ != nullptr) {
     return;
   }

   ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize);

   const Code& trampoline_code = StubCode::FfiCallbackTrampoline();
   const uword code_start = trampoline_code.EntryPoint();
   const uword code_end = code_start + trampoline_code.Size();
   const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1);

   ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(),
                        RXMappingSize());

   // Stub page uses a tight (unaligned) bound for the end of the code area.
   // Otherwise we can read past the end of the code area when doing DuplicateRX.
   stub_page_ = VirtualMemory::ForImagePage(reinterpret_cast<void*>(page_start),
                                            code_end - page_start);

   offset_of_first_trampoline_in_page_ = code_start - page_start;

 #if defined(DART_TARGET_OS_FUCHSIA)
   // On Fuchsia we can't currently duplicate pages, so use the first page of
   // trampolines. Store the stub page's metadata in a separately allocated RW
   // page.
   // TODO(https://dartbug.com/52579): Remove.
   fuchsia_metadata_page_ = VirtualMemory::AllocateAligned(
       MappingSize(), MappingAlignment(), /*is_executable=*/false,
       /*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage");
   Metadata* metadata = reinterpret_cast<Metadata*>(
       fuchsia_metadata_page_->start() + MetadataOffset());
   for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) {
     AddToFreeListLocked(&metadata[i]);
   }
 #endif  // defined(DART_TARGET_OS_FUCHSIA)
 }

 FfiCallbackMetadata::~FfiCallbackMetadata() {
   // Unmap all the trampoline pages. 'VirtualMemory's are new-allocated.
   delete stub_page_;
   for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) {
     delete trampoline_pages_[i];
   }

 #if defined(DART_TARGET_OS_FUCHSIA)
   // TODO(https://dartbug.com/52579): Remove.
   delete fuchsia_metadata_page_;
 #endif  // defined(DART_TARGET_OS_FUCHSIA)
 }

 void FfiCallbackMetadata::Init() {
   ASSERT(singleton_ == nullptr);
   singleton_ = new FfiCallbackMetadata();
 }

 void FfiCallbackMetadata::Cleanup() {
   ASSERT(singleton_ != nullptr);
   delete singleton_;
   singleton_ = nullptr;
 }

 FfiCallbackMetadata* FfiCallbackMetadata::Instance() {
   ASSERT(singleton_ != nullptr);
   return singleton_;
 }

 void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page,
                                               uword index,
                                               void* function) {
   void** slot =
       reinterpret_cast<void**>(page->start() + RuntimeFunctionOffset(index));
   *slot = function;
 }

 VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() {
 #if defined(DART_TARGET_OS_FUCHSIA)
   // TODO(https://dartbug.com/52579): Remove.
   UNREACHABLE();
   return nullptr;
 #else

 #if defined(DART_HOST_OS_MACOS) && !defined(DART_PRECOMPILED_RUNTIME)
   // If we are not going to use vm_remap then we need to pass
   // is_executable=true so that pages get allocated with MAP_JIT flag if
   // necessary. Otherwise OS will kill us with a codesigning violation if
   // hardened runtime is enabled.
   const bool is_executable = true;
 #else
   const bool is_executable = false;
 #endif

   VirtualMemory* new_page = VirtualMemory::AllocateAligned(
       MappingSize(), MappingAlignment(), is_executable,
       /*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage");
   if (new_page == nullptr) {
     return nullptr;
   }

   if (!stub_page_->DuplicateRX(new_page)) {
     delete new_page;
     return nullptr;
   }

 #if !defined(PRODUCT) || defined(FORCE_INCLUDE_DISASSEMBLER)
   if (FLAG_support_disassembler && FLAG_disassemble_stubs) {
     DisassembleToStdout formatter;
     THR_Print("Code for duplicated stub 'FfiCallbackTrampoline' {\n");
     const uword code_start =
         new_page->start() + offset_of_first_trampoline_in_page_;
     Disassembler::Disassemble(code_start, code_start + kPageSize, &formatter,
                               /*comments=*/nullptr);
     THR_Print("}\n");
   }
 #endif

   return new_page;
 #endif  // defined(DART_TARGET_OS_FUCHSIA)
 }

 void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() {
   ASSERT(lock_.IsOwnedByCurrentThread());
   EnsureStubPageLocked();

   if (free_list_head_ != nullptr) {
     return;
   }

   VirtualMemory* new_page = AllocateTrampolinePage();
   if (new_page == nullptr) {
     Exceptions::ThrowOOM();
   }
   trampoline_pages_.Add(new_page);

   // Fill in the runtime functions.
   FillRuntimeFunction(new_page, kGetFfiCallbackMetadata,
                       reinterpret_cast<void*>(DLRT_GetFfiCallbackMetadata));
   FillRuntimeFunction(new_page, kExitTemporaryIsolate,
                       reinterpret_cast<void*>(DLRT_ExitTemporaryIsolate));

   // Add all the trampolines to the free list.
   const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage();
   Metadata* metadata =
       reinterpret_cast<Metadata*>(new_page->start() + MetadataOffset());
   for (intptr_t i = 0; i < trampolines_per_page; ++i) {
     AddToFreeListLocked(&metadata[i]);
   }
 }

 FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateMetadataEntry(
     Isolate* target_isolate,
     TrampolineType trampoline_type,
     uword target_entry_point,
     uint64_t context,
     Metadata** list_head) {
   MutexLocker locker(&lock_);
   EnsureFreeListNotEmptyLocked();
   ASSERT(free_list_head_ != nullptr);
   Metadata* entry = free_list_head_;
   free_list_head_ = entry->free_list_next_;
   if (free_list_head_ == nullptr) {
     ASSERT(free_list_tail_ == entry);
     free_list_tail_ = nullptr;
   }
   Metadata* next_entry = *list_head;
   if (next_entry != nullptr) {
     ASSERT(next_entry->list_prev_ == nullptr);
     next_entry->list_prev_ = entry;
   }
   *entry = Metadata(target_isolate, trampoline_type, target_entry_point,
                     context, nullptr, next_entry);
   *list_head = entry;
   return TrampolineOfMetadata(entry);
 }

 void FfiCallbackMetadata::AddToFreeListLocked(Metadata* entry) {
   ASSERT(lock_.IsOwnedByCurrentThread());
   if (free_list_tail_ == nullptr) {
     ASSERT(free_list_head_ == nullptr);
     free_list_head_ = free_list_tail_ = entry;
   } else {
     ASSERT(free_list_head_ != nullptr && free_list_tail_ != nullptr);
     ASSERT(!free_list_tail_->IsLive());
     free_list_tail_->free_list_next_ = entry;
     free_list_tail_ = entry;
   }
   entry->context_ = 0;
   entry->target_isolate_ = nullptr;
   entry->free_list_next_ = nullptr;
 }

 void FfiCallbackMetadata::DeleteCallbackLocked(Metadata* entry) {
   ASSERT(lock_.IsOwnedByCurrentThread());
   if (entry->trampoline_type_ != TrampolineType::kAsync &&
       entry->context_ != 0) {
     ASSERT(entry->target_isolate_ != nullptr);
     auto* api_state = entry->target_isolate_->group()->api_state();
     ASSERT(api_state != nullptr);
     api_state->FreePersistentHandle(entry->closure_handle());
   }
   AddToFreeListLocked(entry);
 }

 void FfiCallbackMetadata::DeleteAllCallbacks(Metadata** list_head) {
   MutexLocker locker(&lock_);
   for (Metadata* entry = *list_head; entry != nullptr;) {
     Metadata* next = entry->list_next();
     DeleteCallbackLocked(entry);
     entry = next;
   }
   *list_head = nullptr;
 }

 void FfiCallbackMetadata::DeleteCallback(Trampoline trampoline,
                                          Metadata** list_head) {
   MutexLocker locker(&lock_);
   auto* entry = MetadataOfTrampoline(trampoline);
   ASSERT(entry->IsLive());
   auto* prev = entry->list_prev_;
   auto* next = entry->list_next_;
   if (prev != nullptr) {
     prev->list_next_ = next;
   } else {
     ASSERT(*list_head == entry);
     *list_head = next;
   }
   if (next != nullptr) {
     next->list_prev_ = prev;
   }
   DeleteCallbackLocked(entry);
 }

 uword FfiCallbackMetadata::GetEntryPoint(Zone* zone, const Function& function) {
   const auto& code =
       Code::Handle(zone, FLAG_precompiled_mode ? function.CurrentCode()
                                                : function.EnsureHasCode());
   ASSERT(!code.IsNull());
   return code.EntryPoint();
 }

 PersistentHandle* FfiCallbackMetadata::CreatePersistentHandle(
     Isolate* isolate,
     const Closure& closure) {
   auto* api_state = isolate->group()->api_state();
   ASSERT(api_state != nullptr);
   auto* handle = api_state->AllocatePersistentHandle();
   handle->set_ptr(closure);
   return handle;
 }

 FfiCallbackMetadata::Trampoline
 FfiCallbackMetadata::CreateIsolateLocalFfiCallback(Isolate* isolate,
                                                    Zone* zone,
                                                    const Function& function,
                                                    const Closure& closure,
                                                    Metadata** list_head) {
   if (closure.IsNull()) {
     // If the closure is null, it means the target is a static function, so is
     // baked into the trampoline and is an ordinary sync callback.
     ASSERT(function.GetFfiCallbackKind() ==
            FfiCallbackKind::kIsolateLocalStaticCallback);
     return CreateSyncFfiCallbackImpl(isolate, zone, function, nullptr,
                                      list_head);
   } else {
     ASSERT(function.GetFfiCallbackKind() ==
            FfiCallbackKind::kIsolateLocalClosureCallback);
     return CreateSyncFfiCallbackImpl(isolate, zone, function,
                                      CreatePersistentHandle(isolate, closure),
                                      list_head);
   }
 }

 FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateSyncFfiCallbackImpl(
     Isolate* isolate,
     Zone* zone,
     const Function& function,
     PersistentHandle* closure,
     Metadata** list_head) {
   TrampolineType trampoline_type = TrampolineType::kSync;

 #if defined(TARGET_ARCH_IA32)
   // On ia32, store the stack delta that we need to use when returning.
   const intptr_t stack_return_delta =
       function.FfiCSignatureReturnsStruct() && CallingConventions::kUsesRet4
           ? compiler::target::kWordSize
           : 0;
   if (stack_return_delta != 0) {
     ASSERT(stack_return_delta == 4);
     trampoline_type = TrampolineType::kSyncStackDelta4;
   }
 #endif

   return CreateMetadataEntry(isolate, trampoline_type,
                              GetEntryPoint(zone, function),
                              reinterpret_cast<uint64_t>(closure), list_head);
 }

 FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateAsyncFfiCallback(
     Isolate* isolate,
     Zone* zone,
     const Function& send_function,
     Dart_Port send_port,
     Metadata** list_head) {
   ASSERT(send_function.GetFfiCallbackKind() == FfiCallbackKind::kAsyncCallback);
   return CreateMetadataEntry(isolate, TrampolineType::kAsync,
                              GetEntryPoint(zone, send_function),
                              static_cast<uint64_t>(send_port), list_head);
 }

 FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadata(
     Metadata* metadata) const {
   const uword start = MappingStart(reinterpret_cast<uword>(metadata));
   Metadata* metadatas = reinterpret_cast<Metadata*>(start + MetadataOffset());
   const uword index = metadata - metadatas;
 #if defined(DART_TARGET_OS_FUCHSIA)
   return StubCode::FfiCallbackTrampoline().EntryPoint() +
          index * kNativeCallbackTrampolineSize;
 #else
   return start + offset_of_first_trampoline_in_page_ +
          index * kNativeCallbackTrampolineSize;
 #endif
 }

 FfiCallbackMetadata::Metadata* FfiCallbackMetadata::MetadataOfTrampoline(
     Trampoline trampoline) const {
 #if defined(DART_TARGET_OS_FUCHSIA)
   // On Fuchsia the metadata page is separate to the trampoline page.
   // TODO(https://dartbug.com/52579): Remove.
   const uword page_start =
       Utils::RoundDown(trampoline - offset_of_first_trampoline_in_page_,
                        VirtualMemory::PageSize());
   const uword index =
       (trampoline - offset_of_first_trampoline_in_page_ - page_start) /
       kNativeCallbackTrampolineSize;
   ASSERT(index < NumCallbackTrampolinesPerPage());
   Metadata* metadata_table = reinterpret_cast<Metadata*>(
       fuchsia_metadata_page_->start() + MetadataOffset());
   return metadata_table + index;
 #else
   const uword start = MappingStart(trampoline);
   Metadata* metadatas = reinterpret_cast<Metadata*>(start + MetadataOffset());
   const uword index =
       (trampoline - start - offset_of_first_trampoline_in_page_) /
       kNativeCallbackTrampolineSize;
   return &metadatas[index];
 #endif
 }

 FfiCallbackMetadata::Metadata FfiCallbackMetadata::LookupMetadataForTrampoline(
     Trampoline trampoline) const {
   return *MetadataOfTrampoline(trampoline);
 }

 FfiCallbackMetadata* FfiCallbackMetadata::singleton_ = nullptr;

 }  // namespace dart
	// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
	// for details. All rights reserved. Use of this source code is governed by a
	// BSD-style license that can be found in the LICENSE file.

	#include "vm/ffi_callback_metadata.h"

	#include "vm/compiler/assembler/disassembler.h"
	#include "vm/dart_api_state.h"
	#include "vm/flag_list.h"
	#include "vm/object.h"
	#include "vm/runtime_entry.h"
	#include "vm/stub_code.h"

	namespace dart {

	FfiCallbackMetadata::FfiCallbackMetadata() {}

	void FfiCallbackMetadata::EnsureStubPageLocked() {
	ASSERT(lock_.IsOwnedByCurrentThread());
	if (stub_page_ != nullptr) {
	return;
	}

	ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize);

	const Code& trampoline_code = StubCode::FfiCallbackTrampoline();
	const uword code_start = trampoline_code.EntryPoint();
	const uword code_end = code_start + trampoline_code.Size();
	const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1);

	ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(),
	RXMappingSize());

	// Stub page uses a tight (unaligned) bound for the end of the code area.
	// Otherwise we can read past the end of the code area when doing DuplicateRX.
	stub_page_ = VirtualMemory::ForImagePage(reinterpret_cast<void*>(page_start),
	code_end - page_start);

	offset_of_first_trampoline_in_page_ = code_start - page_start;

	#if defined(DART_TARGET_OS_FUCHSIA)
	// On Fuchsia we can't currently duplicate pages, so use the first page of
	// trampolines. Store the stub page's metadata in a separately allocated RW
	// page.
	// TODO(https://dartbug.com/52579): Remove.
	fuchsia_metadata_page_ = VirtualMemory::AllocateAligned(
	MappingSize(), MappingAlignment(), /is_executable=/false,
	/is_compressed=/false, "FfiCallbackMetadata::TrampolinePage");
	Metadata* metadata = reinterpret_cast<Metadata*>(
	fuchsia_metadata_page_->start() + MetadataOffset());
	for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) {
	AddToFreeListLocked(&metadata[i]);
	}
	#endif // defined(DART_TARGET_OS_FUCHSIA)
	}

	FfiCallbackMetadata::~FfiCallbackMetadata() {
	// Unmap all the trampoline pages. 'VirtualMemory's are new-allocated.
	delete stub_page_;
	for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) {
	delete trampoline_pages_[i];
	}

	#if defined(DART_TARGET_OS_FUCHSIA)
	// TODO(https://dartbug.com/52579): Remove.
	delete fuchsia_metadata_page_;
	#endif // defined(DART_TARGET_OS_FUCHSIA)
	}

	void FfiCallbackMetadata::Init() {
	ASSERT(singleton_ == nullptr);
	singleton_ = new FfiCallbackMetadata();
	}

	void FfiCallbackMetadata::Cleanup() {
	ASSERT(singleton_ != nullptr);
	delete singleton_;
	singleton_ = nullptr;
	}

	FfiCallbackMetadata* FfiCallbackMetadata::Instance() {
	ASSERT(singleton_ != nullptr);
	return singleton_;
	}

	void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page,
	uword index,
	void* function) {
	void** slot =
	reinterpret_cast<void**>(page->start() + RuntimeFunctionOffset(index));
	*slot = function;
	}

	VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() {
	#if defined(DART_TARGET_OS_FUCHSIA)
	// TODO(https://dartbug.com/52579): Remove.
	UNREACHABLE();
	return nullptr;
	#else

	#if defined(DART_HOST_OS_MACOS) && !defined(DART_PRECOMPILED_RUNTIME)
	// If we are not going to use vm_remap then we need to pass
	// is_executable=true so that pages get allocated with MAP_JIT flag if
	// necessary. Otherwise OS will kill us with a codesigning violation if
	// hardened runtime is enabled.
	const bool is_executable = true;
	#else
	const bool is_executable = false;
	#endif

	VirtualMemory* new_page = VirtualMemory::AllocateAligned(
	MappingSize(), MappingAlignment(), is_executable,
	/is_compressed=/false, "FfiCallbackMetadata::TrampolinePage");
	if (new_page == nullptr) {
	return nullptr;
	}

	if (!stub_page_->DuplicateRX(new_page)) {
	delete new_page;
	return nullptr;
	}

	#if !defined(PRODUCT) \|\| defined(FORCE_INCLUDE_DISASSEMBLER)
	if (FLAG_support_disassembler && FLAG_disassemble_stubs) {
	DisassembleToStdout formatter;
	THR_Print("Code for duplicated stub 'FfiCallbackTrampoline' {\n");
	const uword code_start =
	new_page->start() + offset_of_first_trampoline_in_page_;
	Disassembler::Disassemble(code_start, code_start + kPageSize, &formatter,
	/comments=/nullptr);
	THR_Print("}\n");
	}
	#endif

	return new_page;
	#endif // defined(DART_TARGET_OS_FUCHSIA)
	}

	void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() {
	ASSERT(lock_.IsOwnedByCurrentThread());
	EnsureStubPageLocked();

	if (free_list_head_ != nullptr) {
	return;
	}

	VirtualMemory* new_page = AllocateTrampolinePage();
	if (new_page == nullptr) {
	Exceptions::ThrowOOM();
	}
	trampoline_pages_.Add(new_page);

	// Fill in the runtime functions.
	FillRuntimeFunction(new_page, kGetFfiCallbackMetadata,
	reinterpret_cast<void*>(DLRT_GetFfiCallbackMetadata));
	FillRuntimeFunction(new_page, kExitTemporaryIsolate,
	reinterpret_cast<void*>(DLRT_ExitTemporaryIsolate));

	// Add all the trampolines to the free list.
	const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage();
	Metadata* metadata =
	reinterpret_cast<Metadata*>(new_page->start() + MetadataOffset());
	for (intptr_t i = 0; i < trampolines_per_page; ++i) {
	AddToFreeListLocked(&metadata[i]);
	}
	}

	FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateMetadataEntry(
	Isolate* target_isolate,
	TrampolineType trampoline_type,
	uword target_entry_point,
	uint64_t context,
	Metadata** list_head) {
	MutexLocker locker(&lock_);
	EnsureFreeListNotEmptyLocked();
	ASSERT(free_list_head_ != nullptr);
	Metadata* entry = free_list_head_;
	free_list_head_ = entry->free_list_next_;
	if (free_list_head_ == nullptr) {
	ASSERT(free_list_tail_ == entry);
	free_list_tail_ = nullptr;
	}
	Metadata* next_entry = *list_head;
	if (next_entry != nullptr) {
	ASSERT(next_entry->list_prev_ == nullptr);
	next_entry->list_prev_ = entry;
	}
	*entry = Metadata(target_isolate, trampoline_type, target_entry_point,
	context, nullptr, next_entry);
	*list_head = entry;
	return TrampolineOfMetadata(entry);
	}

	void FfiCallbackMetadata::AddToFreeListLocked(Metadata* entry) {
	ASSERT(lock_.IsOwnedByCurrentThread());
	if (free_list_tail_ == nullptr) {
	ASSERT(free_list_head_ == nullptr);
	free_list_head_ = free_list_tail_ = entry;
	} else {
	ASSERT(free_list_head_ != nullptr && free_list_tail_ != nullptr);
	ASSERT(!free_list_tail_->IsLive());
	free_list_tail_->free_list_next_ = entry;
	free_list_tail_ = entry;
	}
	entry->context_ = 0;
	entry->target_isolate_ = nullptr;
	entry->free_list_next_ = nullptr;
	}

	void FfiCallbackMetadata::DeleteCallbackLocked(Metadata* entry) {
	ASSERT(lock_.IsOwnedByCurrentThread());
	if (entry->trampoline_type_ != TrampolineType::kAsync &&
	entry->context_ != 0) {
	ASSERT(entry->target_isolate_ != nullptr);
	auto* api_state = entry->target_isolate_->group()->api_state();
	ASSERT(api_state != nullptr);
	api_state->FreePersistentHandle(entry->closure_handle());
	}
	AddToFreeListLocked(entry);
	}

	void FfiCallbackMetadata::DeleteAllCallbacks(Metadata** list_head) {
	MutexLocker locker(&lock_);
	for (Metadata* entry = *list_head; entry != nullptr;) {
	Metadata* next = entry->list_next();
	DeleteCallbackLocked(entry);
	entry = next;
	}
	*list_head = nullptr;
	}

	void FfiCallbackMetadata::DeleteCallback(Trampoline trampoline,
	Metadata** list_head) {
	MutexLocker locker(&lock_);
	auto* entry = MetadataOfTrampoline(trampoline);
	ASSERT(entry->IsLive());
	auto* prev = entry->list_prev_;
	auto* next = entry->list_next_;
	if (prev != nullptr) {
	prev->list_next_ = next;
	} else {
	ASSERT(*list_head == entry);
	*list_head = next;
	}
	if (next != nullptr) {
	next->list_prev_ = prev;
	}
	DeleteCallbackLocked(entry);
	}

	uword FfiCallbackMetadata::GetEntryPoint(Zone* zone, const Function& function) {
	const auto& code =
	Code::Handle(zone, FLAG_precompiled_mode ? function.CurrentCode()
	: function.EnsureHasCode());
	ASSERT(!code.IsNull());
	return code.EntryPoint();
	}

	PersistentHandle* FfiCallbackMetadata::CreatePersistentHandle(
	Isolate* isolate,
	const Closure& closure) {
	auto* api_state = isolate->group()->api_state();
	ASSERT(api_state != nullptr);
	auto* handle = api_state->AllocatePersistentHandle();
	handle->set_ptr(closure);
	return handle;
	}

	FfiCallbackMetadata::Trampoline
	FfiCallbackMetadata::CreateIsolateLocalFfiCallback(Isolate* isolate,
	Zone* zone,
	const Function& function,
	const Closure& closure,
	Metadata** list_head) {
	if (closure.IsNull()) {
	// If the closure is null, it means the target is a static function, so is
	// baked into the trampoline and is an ordinary sync callback.
	ASSERT(function.GetFfiCallbackKind() ==
	FfiCallbackKind::kIsolateLocalStaticCallback);
	return CreateSyncFfiCallbackImpl(isolate, zone, function, nullptr,
	list_head);
	} else {
	ASSERT(function.GetFfiCallbackKind() ==
	FfiCallbackKind::kIsolateLocalClosureCallback);
	return CreateSyncFfiCallbackImpl(isolate, zone, function,
	CreatePersistentHandle(isolate, closure),
	list_head);
	}
	}

	FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateSyncFfiCallbackImpl(
	Isolate* isolate,
	Zone* zone,
	const Function& function,
	PersistentHandle* closure,
	Metadata** list_head) {
	TrampolineType trampoline_type = TrampolineType::kSync;

	#if defined(TARGET_ARCH_IA32)
	// On ia32, store the stack delta that we need to use when returning.
	const intptr_t stack_return_delta =
	function.FfiCSignatureReturnsStruct() && CallingConventions::kUsesRet4
	? compiler::target::kWordSize
	: 0;
	if (stack_return_delta != 0) {
	ASSERT(stack_return_delta == 4);
	trampoline_type = TrampolineType::kSyncStackDelta4;
	}
	#endif

	return CreateMetadataEntry(isolate, trampoline_type,
	GetEntryPoint(zone, function),
	reinterpret_cast<uint64_t>(closure), list_head);
	}

	FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateAsyncFfiCallback(
	Isolate* isolate,
	Zone* zone,
	const Function& send_function,
	Dart_Port send_port,
	Metadata** list_head) {
	ASSERT(send_function.GetFfiCallbackKind() == FfiCallbackKind::kAsyncCallback);
	return CreateMetadataEntry(isolate, TrampolineType::kAsync,
	GetEntryPoint(zone, send_function),
	static_cast<uint64_t>(send_port), list_head);
	}

	FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadata(
	Metadata* metadata) const {
	const uword start = MappingStart(reinterpret_cast<uword>(metadata));
	Metadata* metadatas = reinterpret_cast<Metadata*>(start + MetadataOffset());
	const uword index = metadata - metadatas;
	#if defined(DART_TARGET_OS_FUCHSIA)
	return StubCode::FfiCallbackTrampoline().EntryPoint() +
	index * kNativeCallbackTrampolineSize;
	#else
	return start + offset_of_first_trampoline_in_page_ +
	index * kNativeCallbackTrampolineSize;
	#endif
	}

	FfiCallbackMetadata::Metadata* FfiCallbackMetadata::MetadataOfTrampoline(
	Trampoline trampoline) const {
	#if defined(DART_TARGET_OS_FUCHSIA)
	// On Fuchsia the metadata page is separate to the trampoline page.
	// TODO(https://dartbug.com/52579): Remove.
	const uword page_start =
	Utils::RoundDown(trampoline - offset_of_first_trampoline_in_page_,
	VirtualMemory::PageSize());
	const uword index =
	(trampoline - offset_of_first_trampoline_in_page_ - page_start) /
	kNativeCallbackTrampolineSize;
	ASSERT(index < NumCallbackTrampolinesPerPage());
	Metadata* metadata_table = reinterpret_cast<Metadata*>(
	fuchsia_metadata_page_->start() + MetadataOffset());
	return metadata_table + index;
	#else
	const uword start = MappingStart(trampoline);
	Metadata* metadatas = reinterpret_cast<Metadata*>(start + MetadataOffset());
	const uword index =
	(trampoline - start - offset_of_first_trampoline_in_page_) /
	kNativeCallbackTrampolineSize;
	return &metadatas[index];
	#endif
	}

	FfiCallbackMetadata::Metadata FfiCallbackMetadata::LookupMetadataForTrampoline(
	Trampoline trampoline) const {
	return *MetadataOfTrampoline(trampoline);
	}

	FfiCallbackMetadata* FfiCallbackMetadata::singleton_ = nullptr;

	} // namespace dart