blob: 2d8757daf98a4e88ef067abc8a134752411f2809 [file] [log] [blame]
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/globals.h"
#if defined(DART_HOST_OS_ANDROID) || defined(DART_HOST_OS_LINUX) || \
defined(DART_HOST_OS_MACOS)
#include "vm/virtual_memory.h"
#include <errno.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <unistd.h>
#include "platform/assert.h"
#include "platform/utils.h"
#include "vm/heap/pages.h"
#include "vm/isolate.h"
#include "vm/virtual_memory_compressed.h"
// #define VIRTUAL_MEMORY_LOGGING 1
#if defined(VIRTUAL_MEMORY_LOGGING)
#define LOG_INFO(msg, ...) OS::PrintErr(msg, ##__VA_ARGS__)
#else
#define LOG_INFO(msg, ...)
#endif // defined(VIRTUAL_MEMORY_LOGGING)
namespace dart {
// standard MAP_FAILED causes "error: use of old-style cast" as it
// defines MAP_FAILED as ((void *) -1)
#undef MAP_FAILED
#define MAP_FAILED reinterpret_cast<void*>(-1)
DECLARE_FLAG(bool, dual_map_code);
DECLARE_FLAG(bool, write_protect_code);
#if defined(DART_TARGET_OS_LINUX)
DECLARE_FLAG(bool, generate_perf_events_symbols);
DECLARE_FLAG(bool, generate_perf_jitdump);
#endif
uword VirtualMemory::page_size_ = 0;
static void unmap(uword start, uword end);
static void* GenericMapAligned(void* hint,
int prot,
intptr_t size,
intptr_t alignment,
intptr_t allocated_size,
int map_flags) {
void* address = mmap(hint, allocated_size, prot, map_flags, -1, 0);
LOG_INFO("mmap(%p, 0x%" Px ", %u, ...): %p\n", hint, allocated_size, prot,
address);
if (address == MAP_FAILED) {
return nullptr;
}
const uword base = reinterpret_cast<uword>(address);
const uword aligned_base = Utils::RoundUp(base, alignment);
unmap(base, aligned_base);
unmap(aligned_base + size, base + allocated_size);
return reinterpret_cast<void*>(aligned_base);
}
intptr_t VirtualMemory::CalculatePageSize() {
const intptr_t page_size = getpagesize();
ASSERT(page_size != 0);
ASSERT(Utils::IsPowerOfTwo(page_size));
return page_size;
}
void VirtualMemory::Init() {
#if defined(DART_COMPRESSED_POINTERS)
if (VirtualMemoryCompressedHeap::GetRegion() == nullptr) {
void* address = GenericMapAligned(
nullptr, PROT_NONE, kCompressedHeapSize, kCompressedHeapAlignment,
kCompressedHeapSize + kCompressedHeapAlignment,
MAP_PRIVATE | MAP_ANONYMOUS);
if (address == nullptr) {
int error = errno;
const int kBufferSize = 1024;
char error_buf[kBufferSize];
FATAL2("Failed to reserve region for compressed heap: %d (%s)", error,
Utils::StrError(error, error_buf, kBufferSize));
}
VirtualMemoryCompressedHeap::Init(address);
}
#endif // defined(DART_COMPRESSED_POINTERS)
if (page_size_ != 0) {
// Already initialized.
return;
}
page_size_ = CalculatePageSize();
#if defined(DUAL_MAPPING_SUPPORTED)
// Perf is Linux-specific and the flags aren't defined in Product.
#if defined(DART_TARGET_OS_LINUX) && !defined(PRODUCT)
// Perf interacts strangely with memfds, leading it to sometimes collect
// garbled return addresses.
if (FLAG_generate_perf_events_symbols || FLAG_generate_perf_jitdump) {
LOG_INFO(
"Dual code mapping disabled to generate perf events or jitdump.\n");
FLAG_dual_map_code = false;
return;
}
#endif
// Detect dual mapping exec permission limitation on some platforms,
// such as on docker containers, and disable dual mapping in this case.
// Also detect for missing support of memfd_create syscall.
if (FLAG_dual_map_code) {
intptr_t size = PageSize();
intptr_t alignment = kOldPageSize;
VirtualMemory* vm = AllocateAligned(size, alignment, true, "memfd-test");
if (vm == nullptr) {
LOG_INFO("memfd_create not supported; disabling dual mapping of code.\n");
FLAG_dual_map_code = false;
return;
}
void* region = reinterpret_cast<void*>(vm->region_.start());
void* alias = reinterpret_cast<void*>(vm->alias_.start());
if (region == alias ||
mprotect(region, size, PROT_READ) != 0 || // Remove PROT_WRITE.
mprotect(alias, size, PROT_READ | PROT_EXEC) != 0) { // Add PROT_EXEC.
LOG_INFO("mprotect fails; disabling dual mapping of code.\n");
FLAG_dual_map_code = false;
}
delete vm;
}
#endif // defined(DUAL_MAPPING_SUPPORTED)
#if defined(DART_HOST_OS_LINUX) || defined(DART_HOST_OS_ANDROID)
FILE* fp = fopen("/proc/sys/vm/max_map_count", "r");
if (fp != nullptr) {
size_t max_map_count = 0;
int count = fscanf(fp, "%zu", &max_map_count);
fclose(fp);
if (count == 1) {
size_t max_heap_pages = FLAG_old_gen_heap_size * MB / kOldPageSize;
if (max_map_count < max_heap_pages) {
OS::PrintErr(
"warning: vm.max_map_count (%zu) is not large enough to support "
"--old_gen_heap_size=%d. Consider increasing it with `sysctl -w "
"vm.max_map_count=%zu`\n",
max_map_count, FLAG_old_gen_heap_size, max_heap_pages);
}
}
}
#endif
}
void VirtualMemory::Cleanup() {
#if defined(DART_COMPRESSED_POINTERS)
uword heap_base =
reinterpret_cast<uword>(VirtualMemoryCompressedHeap::GetRegion());
unmap(heap_base, heap_base + kCompressedHeapSize);
VirtualMemoryCompressedHeap::Cleanup();
#endif // defined(DART_COMPRESSED_POINTERS)
}
bool VirtualMemory::DualMappingEnabled() {
return FLAG_dual_map_code;
}
static void unmap(uword start, uword end) {
ASSERT(start <= end);
uword size = end - start;
if (size == 0) {
return;
}
if (munmap(reinterpret_cast<void*>(start), size) != 0) {
int error = errno;
const int kBufferSize = 1024;
char error_buf[kBufferSize];
FATAL2("munmap error: %d (%s)", error,
Utils::StrError(error, error_buf, kBufferSize));
}
}
#if defined(DUAL_MAPPING_SUPPORTED)
// Do not leak file descriptors to child processes.
#if !defined(MFD_CLOEXEC)
#define MFD_CLOEXEC 0x0001U
#endif
// Wrapper to call memfd_create syscall.
static inline int memfd_create(const char* name, unsigned int flags) {
#if !defined(__NR_memfd_create)
errno = ENOSYS;
return -1;
#else
return syscall(__NR_memfd_create, name, flags);
#endif
}
static void* MapAligned(void* hint,
int fd,
int prot,
intptr_t size,
intptr_t alignment,
intptr_t allocated_size) {
void* address =
mmap(hint, allocated_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
LOG_INFO("mmap(%p, 0x%" Px ", PROT_NONE, ...): %p\n", hint, allocated_size,
address);
if (address == MAP_FAILED) {
return nullptr;
}
const uword base = reinterpret_cast<uword>(address);
const uword aligned_base = Utils::RoundUp(base, alignment);
// Guarantee the alignment by mapping at a fixed address inside the above
// mapping. Overlapping region will be automatically discarded in the above
// mapping. Manually discard non-overlapping regions.
address = mmap(reinterpret_cast<void*>(aligned_base), size, prot,
MAP_SHARED | MAP_FIXED, fd, 0);
LOG_INFO("mmap(0x%" Px ", 0x%" Px ", %u, ...): %p\n", aligned_base, size,
prot, address);
if (address == MAP_FAILED) {
unmap(base, base + allocated_size);
return nullptr;
}
ASSERT(address == reinterpret_cast<void*>(aligned_base));
unmap(base, aligned_base);
unmap(aligned_base + size, base + allocated_size);
return address;
}
#endif // defined(DUAL_MAPPING_SUPPORTED)
VirtualMemory* VirtualMemory::AllocateAligned(intptr_t size,
intptr_t alignment,
bool is_executable,
const char* name) {
// When FLAG_write_protect_code is active, code memory (indicated by
// is_executable = true) is allocated as non-executable and later
// changed to executable via VirtualMemory::Protect.
//
// If FLAG_dual_map_code is active, the executable mapping will be mapped RX
// immediately and never changes protection until it is eventually unmapped.
ASSERT(Utils::IsAligned(size, PageSize()));
ASSERT(Utils::IsPowerOfTwo(alignment));
ASSERT(Utils::IsAligned(alignment, PageSize()));
ASSERT(name != nullptr);
#if defined(DART_COMPRESSED_POINTERS)
if (!is_executable) {
MemoryRegion region =
VirtualMemoryCompressedHeap::Allocate(size, alignment);
if (region.pointer() == nullptr) {
return nullptr;
}
mprotect(region.pointer(), region.size(), PROT_READ | PROT_WRITE);
return new VirtualMemory(region, region);
}
#endif // defined(DART_COMPRESSED_POINTERS)
const intptr_t allocated_size = size + alignment - PageSize();
#if defined(DUAL_MAPPING_SUPPORTED)
const bool dual_mapping =
is_executable && FLAG_write_protect_code && FLAG_dual_map_code;
if (dual_mapping) {
int fd = memfd_create(name, MFD_CLOEXEC);
if (fd == -1) {
return nullptr;
}
if (ftruncate(fd, size) == -1) {
close(fd);
return nullptr;
}
const int region_prot = PROT_READ | PROT_WRITE;
void* region_ptr =
MapAligned(nullptr, fd, region_prot, size, alignment, allocated_size);
if (region_ptr == nullptr) {
close(fd);
return nullptr;
}
// The mapping will be RX and stays that way until it will eventually be
// unmapped.
MemoryRegion region(region_ptr, size);
// DUAL_MAPPING_SUPPORTED is false in DART_TARGET_OS_MACOS and hence support
// for MAP_JIT is not required here.
const int alias_prot = PROT_READ | PROT_EXEC;
void* hint = reinterpret_cast<void*>(&Dart_Initialize);
void* alias_ptr =
MapAligned(hint, fd, alias_prot, size, alignment, allocated_size);
close(fd);
if (alias_ptr == nullptr) {
const uword region_base = reinterpret_cast<uword>(region_ptr);
unmap(region_base, region_base + size);
return nullptr;
}
ASSERT(region_ptr != alias_ptr);
MemoryRegion alias(alias_ptr, size);
return new VirtualMemory(region, alias, region);
}
#endif // defined(DUAL_MAPPING_SUPPORTED)
const int prot =
PROT_READ | PROT_WRITE |
((is_executable && !FLAG_write_protect_code) ? PROT_EXEC : 0);
#if defined(DUAL_MAPPING_SUPPORTED)
// Try to use memfd for single-mapped regions too, so they will have an
// associated name for memory attribution. Skip if FLAG_dual_map_code is
// false, which happens if we detected memfd wasn't working in Init above.
if (FLAG_dual_map_code) {
int fd = memfd_create(name, MFD_CLOEXEC);
if (fd == -1) {
return nullptr;
}
if (ftruncate(fd, size) == -1) {
close(fd);
return nullptr;
}
void* region_ptr =
MapAligned(nullptr, fd, prot, size, alignment, allocated_size);
close(fd);
if (region_ptr == nullptr) {
return nullptr;
}
MemoryRegion region(region_ptr, size);
return new VirtualMemory(region, region);
}
#endif
int map_flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if (defined(DART_HOST_OS_MACOS) && !defined(DART_HOST_OS_IOS))
if (is_executable && IsAtLeastOS10_14()) {
map_flags |= MAP_JIT;
}
#endif // defined(DART_HOST_OS_MACOS)
void* hint = nullptr;
// Some 64-bit microarchitectures store only the low 32-bits of targets as
// part of indirect branch prediction, predicting that the target's upper bits
// will be same as the call instruction's address. This leads to misprediction
// for indirect calls crossing a 4GB boundary. We ask mmap to place our
// generated code near the VM binary to avoid this.
if (is_executable) {
hint = reinterpret_cast<void*>(&Dart_Initialize);
}
void* address =
GenericMapAligned(hint, prot, size, alignment, allocated_size, map_flags);
if (address == nullptr) {
return nullptr;
}
MemoryRegion region(reinterpret_cast<void*>(address), size);
return new VirtualMemory(region, region);
}
VirtualMemory::~VirtualMemory() {
#if defined(DART_COMPRESSED_POINTERS)
if (VirtualMemoryCompressedHeap::Contains(reserved_.pointer())) {
madvise(reserved_.pointer(), reserved_.size(), MADV_DONTNEED);
VirtualMemoryCompressedHeap::Free(reserved_.pointer(), reserved_.size());
return;
}
#endif // defined(DART_COMPRESSED_POINTERS)
if (vm_owns_region()) {
unmap(reserved_.start(), reserved_.end());
const intptr_t alias_offset = AliasOffset();
if (alias_offset != 0) {
unmap(reserved_.start() + alias_offset, reserved_.end() + alias_offset);
}
}
}
bool VirtualMemory::FreeSubSegment(void* address, intptr_t size) {
#if defined(DART_COMPRESSED_POINTERS)
// Don't free the sub segment if it's managed by the compressed pointer heap.
if (VirtualMemoryCompressedHeap::Contains(address)) {
return false;
}
#endif // defined(DART_COMPRESSED_POINTERS)
const uword start = reinterpret_cast<uword>(address);
unmap(start, start + size);
return true;
}
void VirtualMemory::Protect(void* address, intptr_t size, Protection mode) {
#if defined(DEBUG)
Thread* thread = Thread::Current();
ASSERT(thread == nullptr || thread->IsMutatorThread() ||
thread->isolate() == nullptr ||
thread->isolate()->mutator_thread()->IsAtSafepoint());
#endif
uword start_address = reinterpret_cast<uword>(address);
uword end_address = start_address + size;
uword page_address = Utils::RoundDown(start_address, PageSize());
int prot = 0;
switch (mode) {
case kNoAccess:
prot = PROT_NONE;
break;
case kReadOnly:
prot = PROT_READ;
break;
case kReadWrite:
prot = PROT_READ | PROT_WRITE;
break;
case kReadExecute:
prot = PROT_READ | PROT_EXEC;
break;
case kReadWriteExecute:
prot = PROT_READ | PROT_WRITE | PROT_EXEC;
break;
}
if (mprotect(reinterpret_cast<void*>(page_address),
end_address - page_address, prot) != 0) {
int error = errno;
const int kBufferSize = 1024;
char error_buf[kBufferSize];
LOG_INFO("mprotect(0x%" Px ", 0x%" Px ", %u) failed\n", page_address,
end_address - page_address, prot);
FATAL2("mprotect error: %d (%s)", error,
Utils::StrError(error, error_buf, kBufferSize));
}
LOG_INFO("mprotect(0x%" Px ", 0x%" Px ", %u) ok\n", page_address,
end_address - page_address, prot);
}
} // namespace dart
#endif // defined(DART_HOST_OS_ANDROID) || defined(DART_HOST_OS_LINUX) || \
// defined(DART_HOST_OS_MACOS)