Google Git
Sign in
dart / sdk / 50c4449e9e79c20cfde52ba8a85d427a539f53c9 / . / runtime / vm / profiler.cc
blob: d8ae4afe7fd21bf59ea2da947019be4548ec80c4 [file] [log] [blame]
// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "platform/address_sanitizer.h"
#include "platform/memory_sanitizer.h"
#include "platform/utils.h"
#include "platform/atomic.h"
#include "vm/allocation.h"
#include "vm/code_patcher.h"
#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/compiler/compiler_state.h"
#endif
#include "vm/debugger.h"
#include "vm/instructions.h"
#include "vm/isolate.h"
#include "vm/json_stream.h"
#include "vm/lockers.h"
#include "vm/message_handler.h"
#include "vm/native_symbol.h"
#include "vm/object.h"
#include "vm/os.h"
#include "vm/profiler.h"
#include "vm/profiler_service.h"
#include "vm/reusable_handles.h"
#include "vm/signal_handler.h"
#include "vm/simulator.h"
#include "vm/stack_frame.h"
#include "vm/timeline.h"
#include "vm/version.h"
namespace dart {
static const intptr_t kMaxSamplesPerTick = 4;
DEFINE_FLAG(bool, trace_profiled_isolates, false, "Trace profiled isolates.");
DEFINE_FLAG(int,
profile_period,
1000,
"Time between profiler samples in microseconds. Minimum 50.");
DEFINE_FLAG(int,
max_profile_depth,
Sample::kPCArraySizeInWords* kMaxSamplesPerTick,
"Maximum number stack frames walked. Minimum 1. Maximum 255.");
#if defined(USING_SIMULATOR)
DEFINE_FLAG(bool, profile_vm, true, "Always collect native stack traces.");
#else
DEFINE_FLAG(bool, profile_vm, false, "Always collect native stack traces.");
#endif
DEFINE_FLAG(bool,
profile_vm_allocation,
false,
"Collect native stack traces when tracing Dart allocations.");
DEFINE_FLAG(
int,
sample_buffer_duration,
0,
"Defines the size of the profiler sample buffer to contain at least "
"N seconds of samples at a given sample rate. If not provided, the "
"default is ~4 seconds. Large values will greatly increase memory "
"consumption.");
// Include native stack dumping helpers into AOT compiler even in PRODUCT
// mode. This allows to report more informative errors when gen_snapshot
// crashes.
#if !defined(PRODUCT) || defined(DART_PRECOMPILER)
ProfilerCounters Profiler::counters_ = {};
static void DumpStackFrame(intptr_t frame_index, uword pc, uword fp) {
uword start = 0;
if (auto const name = NativeSymbolResolver::LookupSymbolName(pc, &start)) {
uword offset = pc - start;
OS::PrintErr(" pc 0x%" Pp " fp 0x%" Pp " %s+0x%" Px "\n", pc, fp, name,
offset);
NativeSymbolResolver::FreeSymbolName(name);
return;
}
char* dso_name;
uword dso_base;
if (NativeSymbolResolver::LookupSharedObject(pc, &dso_base, &dso_name)) {
uword dso_offset = pc - dso_base;
OS::PrintErr(" pc 0x%" Pp " fp 0x%" Pp " %s+0x%" Px "\n", pc, fp, dso_name,
dso_offset);
NativeSymbolResolver::FreeSymbolName(dso_name);
return;
}
OS::PrintErr(" pc 0x%" Pp " fp 0x%" Pp " Unknown symbol\n", pc, fp);
}
class ProfilerStackWalker : public ValueObject {
public:
ProfilerStackWalker(Dart_Port port_id,
Sample* head_sample,
SampleBuffer* sample_buffer,
intptr_t skip_count = 0)
: port_id_(port_id),
sample_(head_sample),
sample_buffer_(sample_buffer),
skip_count_(skip_count),
frames_skipped_(0),
frame_index_(0),
total_frames_(0) {
if (sample_ == NULL) {
ASSERT(sample_buffer_ == NULL);
} else {
ASSERT(sample_buffer_ != NULL);
ASSERT(sample_->head_sample());
}
}
bool Append(uword pc, uword fp) {
if (frames_skipped_ < skip_count_) {
frames_skipped_++;
return true;
}
if (sample_ == NULL) {
DumpStackFrame(frame_index_, pc, fp);
frame_index_++;
total_frames_++;
return true;
}
if (total_frames_ >= FLAG_max_profile_depth) {
sample_->set_truncated_trace(true);
return false;
}
ASSERT(sample_ != NULL);
if (frame_index_ == Sample::kPCArraySizeInWords) {
Sample* new_sample = sample_buffer_->ReserveSampleAndLink(sample_);
if (new_sample == NULL) {
// Could not reserve new sample- mark this as truncated.
sample_->set_truncated_trace(true);
return false;
}
frame_index_ = 0;
sample_ = new_sample;
}
ASSERT(frame_index_ < Sample::kPCArraySizeInWords);
sample_->SetAt(frame_index_, pc);
frame_index_++;
total_frames_++;
return true;
}
protected:
Dart_Port port_id_;
Sample* sample_;
SampleBuffer* sample_buffer_;
intptr_t skip_count_;
intptr_t frames_skipped_;
intptr_t frame_index_;
intptr_t total_frames_;
};
// The layout of C stack frames.
#if defined(HOST_ARCH_IA32) || defined(HOST_ARCH_X64) || \
defined(HOST_ARCH_ARM) || defined(HOST_ARCH_ARM64)
// +-------------+
// | saved IP/LR |
// +-------------+
// | saved FP | <- FP
// +-------------+
static constexpr intptr_t kHostSavedCallerPcSlotFromFp = 1;
static constexpr intptr_t kHostSavedCallerFpSlotFromFp = 0;
#elif defined(HOST_ARCH_RISCV32) || defined(HOST_ARCH_RISCV64)
// +-------------+
// | | <- FP
// +-------------+
// | saved RA |
// +-------------+
// | saved FP |
// +-------------+
static constexpr intptr_t kHostSavedCallerPcSlotFromFp = -1;
static constexpr intptr_t kHostSavedCallerFpSlotFromFp = -2;
#else
#error What architecture?
#endif
// If the VM is compiled without frame pointers (which is the default on
// recent GCC versions with optimizing enabled) the stack walking code may
// fail.
//
class ProfilerNativeStackWalker : public ProfilerStackWalker {
public:
ProfilerNativeStackWalker(ProfilerCounters* counters,
Dart_Port port_id,
Sample* sample,
SampleBuffer* sample_buffer,
uword stack_lower,
uword stack_upper,
uword pc,
uword fp,
uword sp,
intptr_t skip_count = 0)
: ProfilerStackWalker(port_id, sample, sample_buffer, skip_count),
counters_(counters),
stack_upper_(stack_upper),
original_pc_(pc),
original_fp_(fp),
original_sp_(sp),
lower_bound_(stack_lower) {}
void walk() {
const uword kMaxStep = VirtualMemory::PageSize();
Append(original_pc_, original_fp_);
uword* pc = reinterpret_cast<uword*>(original_pc_);
uword* fp = reinterpret_cast<uword*>(original_fp_);
uword* previous_fp = fp;
uword gap = original_fp_ - original_sp_;
if (gap >= kMaxStep) {
// Gap between frame pointer and stack pointer is
// too large.
counters_->incomplete_sample_fp_step.fetch_add(1);
return;
}
if (!ValidFramePointer(fp)) {
counters_->incomplete_sample_fp_bounds.fetch_add(1);
return;
}
while (true) {
pc = CallerPC(fp);
previous_fp = fp;
fp = CallerFP(fp);
if (fp == NULL) {
return;
}
if (fp <= previous_fp) {
// Frame pointer did not move to a higher address.
counters_->incomplete_sample_fp_step.fetch_add(1);
return;
}
gap = fp - previous_fp;
if (gap >= kMaxStep) {
// Frame pointer step is too large.
counters_->incomplete_sample_fp_step.fetch_add(1);
return;
}
if (!ValidFramePointer(fp)) {
// Frame pointer is outside of isolate stack boundary.
counters_->incomplete_sample_fp_bounds.fetch_add(1);
return;
}
const uword pc_value = reinterpret_cast<uword>(pc);
if ((pc_value + 1) < pc_value) {
// It is not uncommon to encounter an invalid pc as we
// traverse a stack frame. Most of these we can tolerate. If
// the pc is so large that adding one to it will cause an
// overflow it is invalid and it will cause headaches later
// while we are building the profile. Discard it.
counters_->incomplete_sample_bad_pc.fetch_add(1);
return;
}
// Move the lower bound up.
lower_bound_ = reinterpret_cast<uword>(fp);
if (!Append(pc_value, reinterpret_cast<uword>(fp))) {
return;
}
}
}
private:
uword* CallerPC(uword* fp) const {
ASSERT(fp != NULL);
uword* caller_pc_ptr = fp + kHostSavedCallerPcSlotFromFp;
// This may actually be uninitialized, by design (see class comment above).
MSAN_UNPOISON(caller_pc_ptr, kWordSize);
ASAN_UNPOISON(caller_pc_ptr, kWordSize);
return reinterpret_cast<uword*>(*caller_pc_ptr);
}
uword* CallerFP(uword* fp) const {
ASSERT(fp != NULL);
uword* caller_fp_ptr = fp + kHostSavedCallerFpSlotFromFp;
// This may actually be uninitialized, by design (see class comment above).
MSAN_UNPOISON(caller_fp_ptr, kWordSize);
ASAN_UNPOISON(caller_fp_ptr, kWordSize);
return reinterpret_cast<uword*>(*caller_fp_ptr);
}
bool ValidFramePointer(uword* fp) const {
if (fp == NULL) {
return false;
}
uword cursor = reinterpret_cast<uword>(fp);
cursor += sizeof(fp);
bool r = (cursor >= lower_bound_) && (cursor < stack_upper_);
return r;
}
ProfilerCounters* const counters_;
const uword stack_upper_;
const uword original_pc_;
const uword original_fp_;
const uword original_sp_;
uword lower_bound_;
};
static bool ValidateThreadStackBounds(uintptr_t fp,
uintptr_t sp,
uword stack_lower,
uword stack_upper) {
if (stack_lower >= stack_upper) {
// Stack boundary is invalid.
return false;
}
if ((sp < stack_lower) || (sp >= stack_upper)) {
// Stack pointer is outside thread's stack boundary.
return false;
}
if ((fp < stack_lower) || (fp >= stack_upper)) {
// Frame pointer is outside threads's stack boundary.
return false;
}
return true;
}
// Get |thread|'s stack boundary and verify that |sp| and |fp| are within
// it. Return |false| if anything looks suspicious.
static bool GetAndValidateThreadStackBounds(OSThread* os_thread,
Thread* thread,
uintptr_t fp,
uintptr_t sp,
uword* stack_lower,
uword* stack_upper) {
ASSERT(os_thread != NULL);
ASSERT(stack_lower != NULL);
ASSERT(stack_upper != NULL);
#if defined(USING_SIMULATOR)
const bool use_simulator_stack_bounds =
thread != NULL && thread->IsExecutingDartCode();
if (use_simulator_stack_bounds) {
Isolate* isolate = thread->isolate();
ASSERT(isolate != NULL);
Simulator* simulator = isolate->simulator();
*stack_lower = simulator->stack_limit();
*stack_upper = simulator->stack_base();
}
#else
const bool use_simulator_stack_bounds = false;
#endif // defined(USING_SIMULATOR)
if (!use_simulator_stack_bounds) {
*stack_lower = os_thread->stack_limit();
*stack_upper = os_thread->stack_base();
}
if ((*stack_lower == 0) || (*stack_upper == 0)) {
return false;
}
if (!use_simulator_stack_bounds && (sp > *stack_lower)) {
// The stack pointer gives us a tighter lower bound.
*stack_lower = sp;
}
return ValidateThreadStackBounds(fp, sp, *stack_lower, *stack_upper);
}
// Some simple sanity checking of |pc|, |fp|, and |sp|.
static bool InitialRegisterCheck(uintptr_t pc, uintptr_t fp, uintptr_t sp) {
if ((sp == 0) || (fp == 0) || (pc == 0)) {
// None of these registers should be zero.
return false;
}
if (sp > fp) {
// Assuming the stack grows down, we should never have a stack pointer above
// the frame pointer.
return false;
}
return true;
}
void Profiler::DumpStackTrace(void* context) {
if (context == NULL) {
DumpStackTrace(/*for_crash=*/true);
return;
}
#if defined(DART_HOST_OS_LINUX) || defined(DART_HOST_OS_MACOS) || \
defined(DART_HOST_OS_ANDROID)
ucontext_t* ucontext = reinterpret_cast<ucontext_t*>(context);
mcontext_t mcontext = ucontext->uc_mcontext;
uword pc = SignalHandler::GetProgramCounter(mcontext);
uword fp = SignalHandler::GetFramePointer(mcontext);
uword sp = SignalHandler::GetCStackPointer(mcontext);
DumpStackTrace(sp, fp, pc, /*for_crash=*/true);
#elif defined(DART_HOST_OS_WINDOWS)
CONTEXT* ctx = reinterpret_cast<CONTEXT*>(context);
#if defined(HOST_ARCH_IA32)
uword pc = static_cast<uword>(ctx->Eip);
uword fp = static_cast<uword>(ctx->Ebp);
uword sp = static_cast<uword>(ctx->Esp);
#elif defined(HOST_ARCH_X64)
uword pc = static_cast<uword>(ctx->Rip);
uword fp = static_cast<uword>(ctx->Rbp);
uword sp = static_cast<uword>(ctx->Rsp);
#else
#error Unsupported architecture.
#endif
DumpStackTrace(sp, fp, pc, /*for_crash=*/true);
#else
// TODO(fschneider): Add support for more platforms.
// Do nothing on unsupported platforms.
#endif
}
void Profiler::DumpStackTrace(bool for_crash) {
uintptr_t sp = OSThread::GetCurrentStackPointer();
uintptr_t fp = 0;
uintptr_t pc = OS::GetProgramCounter();
COPY_FP_REGISTER(fp);
DumpStackTrace(sp, fp, pc, for_crash);
}
void Profiler::DumpStackTrace(uword sp, uword fp, uword pc, bool for_crash) {
if (for_crash) {
// Allow only one stack trace to prevent recursively printing stack traces
// if we hit an assert while printing the stack.
static RelaxedAtomic<uintptr_t> started_dump = 0;
if (started_dump.fetch_add(1u) != 0) {
OS::PrintErr("Aborting re-entrant request for stack trace.\n");
return;
}
}
auto os_thread = OSThread::Current();
ASSERT(os_thread != nullptr);
auto thread = Thread::Current(); // NULL if no current isolate.
auto isolate = thread == nullptr ? nullptr : thread->isolate();
auto isolate_group = thread == nullptr ? nullptr : thread->isolate_group();
auto source = isolate_group == nullptr ? nullptr : isolate_group->source();
auto vm_source =
Dart::vm_isolate() == nullptr ? nullptr : Dart::vm_isolate()->source();
const char* isolate_group_name =
isolate_group == nullptr ? "(nil)" : isolate_group->source()->name;
const char* isolate_name = isolate == nullptr ? "(nil)" : isolate->name();
#if defined(PRODUCT)
const intptr_t thread_id = -1;
#else
const intptr_t thread_id = OSThread::ThreadIdToIntPtr(os_thread->trace_id());
#endif
OS::PrintErr("version=%s\n", Version::String());
OS::PrintErr("pid=%" Pd ", thread=%" Pd
", isolate_group=%s(%p), isolate=%s(%p)\n",
static_cast<intptr_t>(OS::ProcessId()), thread_id,
isolate_group_name, isolate_group, isolate_name, isolate);
OS::PrintErr("isolate_instructions=%" Px ", vm_instructions=%" Px "\n",
source == nullptr
? 0
: reinterpret_cast<uword>(source->snapshot_instructions),
vm_source == nullptr
? 0
: reinterpret_cast<uword>(vm_source->snapshot_instructions));
if (!InitialRegisterCheck(pc, fp, sp)) {
OS::PrintErr("Stack dump aborted because InitialRegisterCheck failed.\n");
return;
}
uword stack_lower = 0;
uword stack_upper = 0;
if (!GetAndValidateThreadStackBounds(os_thread, thread, fp, sp, &stack_lower,
&stack_upper)) {
OS::PrintErr(
"Stack dump aborted because GetAndValidateThreadStackBounds failed.\n");
return;
}
ProfilerNativeStackWalker native_stack_walker(&counters_, ILLEGAL_PORT, NULL,
NULL, stack_lower, stack_upper,
pc, fp, sp,
/*skip_count=*/0);
native_stack_walker.walk();
OS::PrintErr("-- End of DumpStackTrace\n");
if (thread != nullptr) {
if (thread->execution_state() == Thread::kThreadInNative) {
TransitionNativeToVM transition(thread);
StackFrame::DumpCurrentTrace();
} else if (thread->execution_state() == Thread::kThreadInVM) {
StackFrame::DumpCurrentTrace();
#if !defined(DART_PRECOMPILED_RUNTIME)
if (thread->HasCompilerState()) {
thread->compiler_state().ReportCrash();
}
#endif
}
}
}
#endif // !defined(PRODUCT) || defined(DART_PRECOMPILER)
#ifndef PRODUCT
RelaxedAtomic<bool> Profiler::initialized_ = false;
SampleBlockBuffer* Profiler::sample_block_buffer_ = nullptr;
AllocationSampleBuffer* Profiler::allocation_sample_buffer_ = nullptr;
bool SampleBlockProcessor::initialized_ = false;
bool SampleBlockProcessor::shutdown_ = false;
bool SampleBlockProcessor::thread_running_ = false;
ThreadJoinId SampleBlockProcessor::processor_thread_id_ =
OSThread::kInvalidThreadJoinId;
Monitor* SampleBlockProcessor::monitor_ = nullptr;
void Profiler::Init() {
// Place some sane restrictions on user controlled flags.
SetSampleDepth(FLAG_max_profile_depth);
if (!FLAG_profiler) {
return;
}
ASSERT(!initialized_);
SetSamplePeriod(FLAG_profile_period);
// The profiler may have been shutdown previously, in which case the sample
// buffer will have already been initialized.
if (sample_block_buffer_ == nullptr) {
intptr_t num_blocks = CalculateSampleBufferCapacity();
sample_block_buffer_ = new SampleBlockBuffer(num_blocks);
Profiler::InitAllocationSampleBuffer();
}
ThreadInterrupter::Init();
ThreadInterrupter::Startup();
SampleBlockProcessor::Init();
SampleBlockProcessor::Startup();
initialized_ = true;
}
void Profiler::InitAllocationSampleBuffer() {
ASSERT(Profiler::allocation_sample_buffer_ == NULL);
if (FLAG_profiler_native_memory) {
Profiler::allocation_sample_buffer_ = new AllocationSampleBuffer();
}
}
class SampleBlockCleanupVisitor : public IsolateVisitor {
public:
SampleBlockCleanupVisitor() = default;
virtual ~SampleBlockCleanupVisitor() = default;
void VisitIsolate(Isolate* isolate) {
isolate->set_current_allocation_sample_block(nullptr);
{
MutexLocker ml(isolate->current_sample_block_lock());
isolate->set_current_sample_block(nullptr);
}
}
};
void Profiler::Cleanup() {
if (!FLAG_profiler) {
return;
}
ASSERT(initialized_);
ThreadInterrupter::Cleanup();
SampleBlockProcessor::Cleanup();
SampleBlockCleanupVisitor visitor;
Isolate::VisitIsolates(&visitor);
initialized_ = false;
}
void Profiler::UpdateRunningState() {
if (!FLAG_profiler && initialized_) {
Cleanup();
} else if (FLAG_profiler && !initialized_) {
Init();
}
}
void Profiler::SetSampleDepth(intptr_t depth) {
const int kMinimumDepth = 2;
const int kMaximumDepth = 255;
if (depth < kMinimumDepth) {
FLAG_max_profile_depth = kMinimumDepth;
} else if (depth > kMaximumDepth) {
FLAG_max_profile_depth = kMaximumDepth;
} else {
FLAG_max_profile_depth = depth;
}
}
static intptr_t SamplesPerSecond() {
const intptr_t kMicrosPerSec = 1000000;
return kMicrosPerSec / FLAG_profile_period;
}
intptr_t Profiler::CalculateSampleBufferCapacity() {
if (FLAG_sample_buffer_duration <= 0) {
return SampleBlockBuffer::kDefaultBlockCount;
}
// Deeper stacks require more than a single Sample object to be represented
// correctly. These samples are chained, so we need to determine the worst
// case sample chain length for a single stack.
const intptr_t max_sample_chain_length =
FLAG_max_profile_depth / kMaxSamplesPerTick;
const intptr_t sample_count = FLAG_sample_buffer_duration *
SamplesPerSecond() * max_sample_chain_length;
return (sample_count / SampleBlock::kSamplesPerBlock) + 1;
}
void Profiler::SetSamplePeriod(intptr_t period) {
const int kMinimumProfilePeriod = 50;
if (period < kMinimumProfilePeriod) {
FLAG_profile_period = kMinimumProfilePeriod;
} else {
FLAG_profile_period = period;
}
ThreadInterrupter::SetInterruptPeriod(FLAG_profile_period);
}
void Profiler::UpdateSamplePeriod() {
SetSamplePeriod(FLAG_profile_period);
}
SampleBlockBuffer::SampleBlockBuffer(intptr_t blocks,
intptr_t samples_per_block) {
const intptr_t size = Utils::RoundUp(
blocks * samples_per_block * sizeof(Sample), VirtualMemory::PageSize());
const bool executable = false;
const bool compressed = false;
memory_ =
VirtualMemory::Allocate(size, executable, compressed, "dart-profiler");
if (memory_ == NULL) {
OUT_OF_MEMORY();
}
sample_buffer_ = reinterpret_cast<Sample*>(memory_->address());
blocks_ = new SampleBlock[blocks];
for (intptr_t i = 0; i < blocks; ++i) {
blocks_[i].Init(&sample_buffer_[i * samples_per_block], samples_per_block);
}
capacity_ = blocks;
cursor_ = 0;
free_list_head_ = nullptr;
free_list_tail_ = nullptr;
}
SampleBlockBuffer::~SampleBlockBuffer() {
delete[] blocks_;
blocks_ = nullptr;
delete memory_;
memory_ = nullptr;
capacity_ = 0;
cursor_ = 0;
}
SampleBlock* SampleBlockBuffer::ReserveSampleBlock() {
// Don't increment right away to avoid unlikely wrap-around errors.
if (cursor_.load() < capacity_) {
intptr_t index = cursor_.fetch_add(1u);
// Check the index again to make sure the last block hasn't been snatched
// from underneath us.
if (index < capacity_) {
return &blocks_[index];
}
}
// Try to re-use a previously freed SampleBlock once we've handed out each
// block at least once. Freed blocks aren't cleared immediately and are still
// valid until they're re-allocated, similar to how a ring buffer would clear
// the oldest samples.
SampleBlock* block = GetFreeBlock();
if (block != nullptr) {
block->Clear();
}
return block;
}
void SampleBlockBuffer::ProcessCompletedBlocks() {
Thread* thread = Thread::Current();
DisableThreadInterruptsScope dtis(thread);
int64_t start = Dart_TimelineGetMicros();
thread->isolate()->ProcessFreeSampleBlocks(thread);
int64_t end = Dart_TimelineGetMicros();
Dart_TimelineEvent("SampleBlockBuffer::ProcessCompletedBlocks", start, end,
Dart_Timeline_Event_Duration, 0, nullptr, nullptr);
}
ProcessedSampleBuffer* SampleBlockListProcessor::BuildProcessedSampleBuffer(
SampleFilter* filter,
ProcessedSampleBuffer* buffer) {
Thread* thread = Thread::Current();
Zone* zone = thread->zone();
if (buffer == nullptr) {
buffer = new (zone) ProcessedSampleBuffer();
}
while (head_ != nullptr) {
head_->BuildProcessedSampleBuffer(filter, buffer);
head_ = head_->next_free_;
}
return buffer;
}
bool SampleBlockListProcessor::HasStreamableSamples(Thread* thread) {
ReusableGrowableObjectArrayHandleScope reusable_array_handle_scope(thread);
Zone* zone = thread->zone();
Isolate* isolate = thread->isolate();
ASSERT(isolate->tag_table() != GrowableObjectArray::null());
GrowableObjectArray& tag_table = reusable_array_handle_scope.Handle();
tag_table ^= isolate->tag_table();
UserTag& tag = UserTag::Handle(zone);
while (head_ != nullptr) {
if (head_->HasStreamableSamples(tag_table, &tag)) {
return true;
}
head_ = head_->next_free_;
}
return false;
}
bool SampleBlock::HasStreamableSamples(const GrowableObjectArray& tag_table,
UserTag* tag) {
for (intptr_t i = 0; i < capacity_; ++i) {
Sample* sample = At(i);
uword sample_tag = sample->user_tag();
for (intptr_t j = 0; j < tag_table.Length(); ++j) {
*tag ^= tag_table.At(j);
if (tag->tag() == sample_tag && tag->streamable()) {
return true;
}
}
}
return false;
}
ProcessedSampleBuffer* SampleBlockBuffer::BuildProcessedSampleBuffer(
SampleFilter* filter,
ProcessedSampleBuffer* buffer) {
Thread* thread = Thread::Current();
Zone* zone = thread->zone();
if (buffer == nullptr) {
buffer = new (zone) ProcessedSampleBuffer();
}
for (intptr_t i = 0; i < capacity_; ++i) {
(&blocks_[i])->BuildProcessedSampleBuffer(filter, buffer);
}
return buffer;
}
Sample* SampleBlock::ReserveSample() {
if (full_.load()) {
return nullptr;
}
intptr_t slot = cursor_.fetch_add(1u);
if (slot + 1 == capacity_) {
full_ = true;
}
return (slot < capacity_) ? At(slot) : nullptr;
}
Sample* SampleBlock::ReserveSampleAndLink(Sample* previous) {
ASSERT(previous != nullptr);
SampleBlockBuffer* buffer = Profiler::sample_block_buffer();
Isolate* isolate = owner_;
ASSERT(isolate != nullptr);
Sample* next = previous->is_allocation_sample()
? buffer->ReserveAllocationSample(isolate)
: buffer->ReserveCPUSample(isolate);
if (next == nullptr) {
return nullptr; // No blocks left, so drop sample.
}
next->Init(previous->port(), previous->timestamp(), previous->tid());
next->set_head_sample(false);
// Mark that previous continues at next.
previous->SetContinuation(next);
return next;
}
Sample* SampleBlockBuffer::ReserveCPUSample(Isolate* isolate) {
return ReserveSampleImpl(isolate, false);
}
Sample* SampleBlockBuffer::ReserveAllocationSample(Isolate* isolate) {
return ReserveSampleImpl(isolate, true);
}
Sample* SampleBlockBuffer::ReserveSampleImpl(Isolate* isolate,
bool allocation_sample) {
SampleBlock* block = allocation_sample
? isolate->current_allocation_sample_block()
: isolate->current_sample_block();
Sample* sample = nullptr;
if (block != nullptr) {
sample = block->ReserveSample();
}
if (sample != nullptr) {
return sample;
}
SampleBlock* next = nullptr;
if (allocation_sample) {
// We only need to be locked while accessing the CPU sample block since
// Dart allocations can only occur on the mutator thread.
next = ReserveSampleBlock();
if (next == nullptr) {
// We're out of blocks to reserve. Drop the sample.
return nullptr;
}
isolate->set_current_allocation_sample_block(next);
isolate->FreeSampleBlock(block);
} else {
MutexLocker locker(isolate->current_sample_block_lock());
next = ReserveSampleBlock();
if (next == nullptr) {
// We're out of blocks to reserve. Drop the sample.
return nullptr;
}
isolate->set_current_sample_block(next);
isolate->FreeSampleBlock(block);
}
next->set_is_allocation_block(allocation_sample);
bool scheduled = can_process_block_.exchange(true);
if (!scheduled) {
isolate->mutator_thread()->ScheduleInterrupts(Thread::kVMInterrupt);
}
return ReserveSampleImpl(isolate, allocation_sample);
}
AllocationSampleBuffer::AllocationSampleBuffer(intptr_t capacity) {
const intptr_t size =
Utils::RoundUp(capacity * sizeof(Sample), VirtualMemory::PageSize());
const bool executable = false;
const bool compressed = false;
memory_ =
VirtualMemory::Allocate(size, executable, compressed, "dart-profiler");
if (memory_ == NULL) {
OUT_OF_MEMORY();
}
Init(reinterpret_cast<Sample*>(memory_->address()), capacity);
free_sample_list_ = nullptr;
cursor_ = 0;
}
AllocationSampleBuffer::~AllocationSampleBuffer() {
delete memory_;
memory_ = nullptr;
}
void AllocationSampleBuffer::FreeAllocationSample(Sample* sample) {
MutexLocker ml(&mutex_);
while (sample != nullptr) {
Sample* next = sample->continuation_sample();
sample->Clear();
sample->set_next_free(free_sample_list_);
free_sample_list_ = sample;
sample = next;
}
}
intptr_t AllocationSampleBuffer::ReserveSampleSlotLocked() {
if (free_sample_list_ != NULL) {
Sample* free_sample = free_sample_list_;
free_sample_list_ = free_sample->next_free();
free_sample->set_next_free(NULL);
uint8_t* samples_array_ptr = reinterpret_cast<uint8_t*>(samples_);
uint8_t* free_sample_ptr = reinterpret_cast<uint8_t*>(free_sample);
return static_cast<intptr_t>((free_sample_ptr - samples_array_ptr) /
sizeof(Sample));
} else if (cursor_ < static_cast<intptr_t>(capacity_ - 1)) {
return cursor_ += 1;
} else {
return -1;
}
}
Sample* AllocationSampleBuffer::ReserveSampleAndLink(Sample* previous) {
MutexLocker ml(&mutex_);
ASSERT(previous != NULL);
intptr_t next_index = ReserveSampleSlotLocked();
if (next_index < 0) {
// Could not find a free sample.
return NULL;
}
Sample* next = At(next_index);
next->Init(previous->port(), previous->timestamp(), previous->tid());
next->set_native_allocation_address(previous->native_allocation_address());
next->set_native_allocation_size_bytes(
previous->native_allocation_size_bytes());
next->set_head_sample(false);
// Mark that previous continues at next.
previous->SetContinuation(next);
return next;
}
Sample* AllocationSampleBuffer::ReserveSample() {
MutexLocker ml(&mutex_);
intptr_t index = ReserveSampleSlotLocked();
if (index < 0) {
return NULL;
}
return At(index);
}
// Attempts to find the true return address when a Dart frame is being setup
// or torn down.
// NOTE: Architecture specific implementations below.
class ReturnAddressLocator : public ValueObject {
public:
ReturnAddressLocator(Sample* sample, const Code& code)
: stack_buffer_(sample->GetStackBuffer()),
pc_(sample->pc()),
code_(Code::ZoneHandle(code.ptr())) {
ASSERT(!code_.IsNull());
ASSERT(code_.ContainsInstructionAt(pc()));
}
ReturnAddressLocator(uword pc, uword* stack_buffer, const Code& code)
: stack_buffer_(stack_buffer),
pc_(pc),
code_(Code::ZoneHandle(code.ptr())) {
ASSERT(!code_.IsNull());
ASSERT(code_.ContainsInstructionAt(pc_));
}
uword pc() { return pc_; }
// Returns false on failure.
bool LocateReturnAddress(uword* return_address);
// Returns offset into code object.
intptr_t RelativePC() {
ASSERT(pc() >= code_.PayloadStart());
return static_cast<intptr_t>(pc() - code_.PayloadStart());
}
uint8_t* CodePointer(intptr_t offset) {
const intptr_t size = code_.Size();
ASSERT(offset < size);
uint8_t* code_pointer = reinterpret_cast<uint8_t*>(code_.PayloadStart());
code_pointer += offset;
return code_pointer;
}
uword StackAt(intptr_t i) {
ASSERT(i >= 0);
ASSERT(i < Sample::kStackBufferSizeInWords);
return stack_buffer_[i];
}
private:
uword* stack_buffer_;
uword pc_;
const Code& code_;
};
#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64)
bool ReturnAddressLocator::LocateReturnAddress(uword* return_address) {
ASSERT(return_address != NULL);
const intptr_t offset = RelativePC();
ASSERT(offset >= 0);
const intptr_t size = code_.Size();
ASSERT(offset < size);
const intptr_t prologue_offset = code_.GetPrologueOffset();
if (offset < prologue_offset) {
// Before the prologue, return address is at the top of the stack.
// TODO(johnmccutchan): Some intrinsics and stubs do not conform to the
// expected stack layout. Use a more robust solution for those code objects.
*return_address = StackAt(0);
return true;
}
// Detect if we are:
// push ebp <--- here
// mov ebp, esp
// on X64 the register names are different but the sequence is the same.
ProloguePattern pp(pc());
if (pp.IsValid()) {
// Stack layout:
// 0 RETURN ADDRESS.
*return_address = StackAt(0);
return true;
}
// Detect if we are:
// push ebp
// mov ebp, esp <--- here
// on X64 the register names are different but the sequence is the same.
SetFramePointerPattern sfpp(pc());
if (sfpp.IsValid()) {
// Stack layout:
// 0 CALLER FRAME POINTER
// 1 RETURN ADDRESS
*return_address = StackAt(1);
return true;
}
// Detect if we are:
// ret <--- here
ReturnPattern rp(pc());
if (rp.IsValid()) {
// Stack layout:
// 0 RETURN ADDRESS.
*return_address = StackAt(0);
return true;
}
return false;
}
#elif defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64) || \
defined(TARGET_ARCH_RISCV32) || defined(TARGET_ARCH_RISCV64)
bool ReturnAddressLocator::LocateReturnAddress(uword* return_address) {
ASSERT(return_address != NULL);
return false;
}
#else
#error ReturnAddressLocator implementation missing for this architecture.
#endif
bool SampleFilter::TimeFilterSample(Sample* sample) {
if ((time_origin_micros_ == -1) || (time_extent_micros_ == -1)) {
// No time filter passed in, always pass.
return true;
}
const int64_t timestamp = sample->timestamp();
int64_t delta = timestamp - time_origin_micros_;
return (delta >= 0) && (delta <= time_extent_micros_);
}
bool SampleFilter::TaskFilterSample(Sample* sample) {
const intptr_t task = static_cast<intptr_t>(sample->thread_task());
if (thread_task_mask_ == kNoTaskFilter) {
return true;
}
return (task & thread_task_mask_) != 0;
}
ClearProfileVisitor::ClearProfileVisitor(Isolate* isolate)
: SampleVisitor(isolate->main_port()) {}
void ClearProfileVisitor::VisitSample(Sample* sample) {
sample->Clear();
}
// Executing Dart code, walk the stack.
class ProfilerDartStackWalker : public ProfilerStackWalker {
public:
ProfilerDartStackWalker(Thread* thread,
Sample* sample,
SampleBuffer* sample_buffer,
uword pc,
uword fp,
uword sp,
uword lr,
bool allocation_sample,
intptr_t skip_count = 0)
: ProfilerStackWalker((thread->isolate() != NULL)
? thread->isolate()->main_port()
: ILLEGAL_PORT,
sample,
sample_buffer,
skip_count),
thread_(thread),
pc_(reinterpret_cast<uword*>(pc)),
fp_(reinterpret_cast<uword*>(fp)),
sp_(reinterpret_cast<uword*>(sp)),
lr_(reinterpret_cast<uword*>(lr)) {}
void walk() {
RELEASE_ASSERT(StubCode::HasBeenInitialized());
if (thread_->isolate()->IsDeoptimizing()) {
sample_->set_ignore_sample(true);
return;
}
uword* exit_fp = reinterpret_cast<uword*>(thread_->top_exit_frame_info());
bool has_exit_frame = exit_fp != 0;
if (has_exit_frame) {
// Exited from compiled code.
pc_ = 0;
fp_ = exit_fp;
// Skip exit frame.
pc_ = CallerPC();
fp_ = CallerFP();
} else {
if (thread_->vm_tag() == VMTag::kDartTagId) {
// Running compiled code.
// Use the FP and PC from the thread interrupt or simulator; already set
// in the constructor.
} else {
// No Dart on the stack; caller shouldn't use this walker.
UNREACHABLE();
}
const bool is_entry_frame =
#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64)
StubCode::InInvocationStub(Stack(0)) ||
StubCode::InInvocationStub(Stack(1));
#else
StubCode::InInvocationStub(reinterpret_cast<uword>(lr_));
#endif
if (is_entry_frame) {
// During the prologue of a function, CallerPC will return the caller's
// caller. For most frames, the missing PC will be added during profile
// processing. However, during this stack walk, it can cause us to fail
// to identify the entry frame and lead the stack walk into the weeds.
// Do not continue the stalk walk since this might be a false positive
// from a Smi or unboxed value.
sample_->set_ignore_sample(true);
return;
}
}
sample_->set_exit_frame_sample(has_exit_frame);
for (;;) {
// Skip entry frame.
if (StubCode::InInvocationStub(reinterpret_cast<uword>(pc_))) {
pc_ = 0;
fp_ = ExitLink();
if (fp_ == 0) {
break; // End of Dart stack.
}
// Skip exit frame.
pc_ = CallerPC();
fp_ = CallerFP();
// At least one frame between exit and next entry frame.
RELEASE_ASSERT(
!StubCode::InInvocationStub(reinterpret_cast<uword>(pc_)));
}
if (!Append(reinterpret_cast<uword>(pc_), reinterpret_cast<uword>(fp_))) {
break; // Sample is full.
}
pc_ = CallerPC();
fp_ = CallerFP();
}
}
private:
uword* CallerPC() const {
ASSERT(fp_ != NULL);
uword* caller_pc_ptr = fp_ + kSavedCallerPcSlotFromFp;
// MSan/ASan are unaware of frames initialized by generated code.
MSAN_UNPOISON(caller_pc_ptr, kWordSize);
ASAN_UNPOISON(caller_pc_ptr, kWordSize);
return reinterpret_cast<uword*>(*caller_pc_ptr);
}
uword* CallerFP() const {
ASSERT(fp_ != NULL);
uword* caller_fp_ptr = fp_ + kSavedCallerFpSlotFromFp;
// MSan/ASan are unaware of frames initialized by generated code.
MSAN_UNPOISON(caller_fp_ptr, kWordSize);
ASAN_UNPOISON(caller_fp_ptr, kWordSize);
return reinterpret_cast<uword*>(*caller_fp_ptr);
}
uword* ExitLink() const {
ASSERT(fp_ != NULL);
uword* exit_link_ptr = fp_ + kExitLinkSlotFromEntryFp;
// MSan/ASan are unaware of frames initialized by generated code.
MSAN_UNPOISON(exit_link_ptr, kWordSize);
ASAN_UNPOISON(exit_link_ptr, kWordSize);
return reinterpret_cast<uword*>(*exit_link_ptr);
}
uword Stack(intptr_t index) const {
ASSERT(sp_ != NULL);
uword* stack_ptr = sp_ + index;
// MSan/ASan are unaware of frames initialized by generated code.
MSAN_UNPOISON(stack_ptr, kWordSize);
ASAN_UNPOISON(stack_ptr, kWordSize);
return *stack_ptr;
}
Thread* const thread_;
uword* pc_;
uword* fp_;
uword* sp_;
uword* lr_;
};
static void CopyStackBuffer(Sample* sample, uword sp_addr) {
ASSERT(sample != NULL);
uword* sp = reinterpret_cast<uword*>(sp_addr);
uword* buffer = sample->GetStackBuffer();
if (sp != NULL) {
for (intptr_t i = 0; i < Sample::kStackBufferSizeInWords; i++) {
MSAN_UNPOISON(sp, kWordSize);
ASAN_UNPOISON(sp, kWordSize);
buffer[i] = *sp;
sp++;
}
}
}
#if defined(DART_HOST_OS_WINDOWS)
// On Windows this code is synchronously executed from the thread interrupter
// thread. This means we can safely have a static fault_address.
static uword fault_address = 0;
static LONG GuardPageExceptionFilter(EXCEPTION_POINTERS* ep) {
fault_address = 0;
if (ep->ExceptionRecord->ExceptionCode != STATUS_GUARD_PAGE_VIOLATION) {
return EXCEPTION_CONTINUE_SEARCH;
}
// https://goo.gl/p5Fe10
fault_address = ep->ExceptionRecord->ExceptionInformation[1];
// Read access.
ASSERT(ep->ExceptionRecord->ExceptionInformation[0] == 0);
return EXCEPTION_EXECUTE_HANDLER;
}
#endif
// All memory access done to collect the sample is performed in CollectSample.
static void CollectSample(Isolate* isolate,
bool exited_dart_code,
bool in_dart_code,
Sample* sample,
ProfilerNativeStackWalker* native_stack_walker,
ProfilerDartStackWalker* dart_stack_walker,
uword pc,
uword fp,
uword sp,
ProfilerCounters* counters) {
ASSERT(counters != NULL);
#if defined(DART_HOST_OS_WINDOWS)
// Use structured exception handling to trap guard page access on Windows.
__try {
#endif
if (in_dart_code) {
// We can only trust the stack pointer if we are executing Dart code.
// See http://dartbug.com/20421 for details.
CopyStackBuffer(sample, sp);
}
if (FLAG_profile_vm) {
// Always walk the native stack collecting both native and Dart frames.
counters->stack_walker_native.fetch_add(1);
native_stack_walker->walk();
} else if (StubCode::HasBeenInitialized() && exited_dart_code) {
counters->stack_walker_dart_exit.fetch_add(1);
// We have a valid exit frame info, use the Dart stack walker.
dart_stack_walker->walk();
} else if (StubCode::HasBeenInitialized() && in_dart_code) {
counters->stack_walker_dart.fetch_add(1);
// We are executing Dart code. We have frame pointers.
dart_stack_walker->walk();
} else {
counters->stack_walker_none.fetch_add(1);
sample->SetAt(0, pc);
}
#if defined(DART_HOST_OS_WINDOWS)
// Use structured exception handling to trap guard page access.
} __except (GuardPageExceptionFilter(GetExceptionInformation())) { // NOLINT
// Sample collection triggered a guard page fault:
// 1) discard entire sample.
sample->set_ignore_sample(true);
// 2) Reenable guard bit on page that triggered the fault.
// https://goo.gl/5mCsXW
DWORD new_protect = PAGE_READWRITE | PAGE_GUARD;
DWORD old_protect = 0;
BOOL success =
VirtualProtect(reinterpret_cast<void*>(fault_address),
sizeof(fault_address), new_protect, &old_protect);
USE(success);
ASSERT(success);
ASSERT(old_protect == PAGE_READWRITE);
}
#endif
}
static Sample* SetupSample(Thread* thread,
bool allocation_sample,
ThreadId tid) {
ASSERT(thread != NULL);
Isolate* isolate = thread->isolate();
SampleBlockBuffer* buffer = Profiler::sample_block_buffer();
Sample* sample = allocation_sample ? buffer->ReserveAllocationSample(isolate)
: buffer->ReserveCPUSample(isolate);
if (sample == nullptr) {
return nullptr;
}
sample->Init(isolate->main_port(), OS::GetCurrentMonotonicMicros(), tid);
uword vm_tag = thread->vm_tag();
#if defined(USING_SIMULATOR)
// When running in the simulator, the runtime entry function address
// (stored as the vm tag) is the address of a redirect function.
// Attempt to find the real runtime entry function address and use that.
uword redirect_vm_tag = Simulator::FunctionForRedirect(vm_tag);
if (redirect_vm_tag != 0) {
vm_tag = redirect_vm_tag;
}
#endif
sample->set_vm_tag(vm_tag);
sample->set_user_tag(isolate->user_tag());
sample->set_thread_task(thread->task_kind());
return sample;
}
static Sample* SetupSampleNative(AllocationSampleBuffer* sample_buffer,
ThreadId tid) {
Sample* sample = sample_buffer->ReserveSample();
if (sample == NULL) {
return NULL;
}
sample->Init(ILLEGAL_PORT, OS::GetCurrentMonotonicMicros(), tid);
Thread* thread = Thread::Current();
// Note: setting thread task in order to be consistent with other samples. The
// task kind is not used by NativeAllocationSampleFilter for filtering
// purposes as some samples may be collected when no thread exists.
if (thread != NULL) {
sample->set_thread_task(thread->task_kind());
sample->set_vm_tag(thread->vm_tag());
} else {
sample->set_vm_tag(VMTag::kEmbedderTagId);
}
return sample;
}
static bool CheckIsolate(Isolate* isolate) {
if ((isolate == NULL) || (Dart::vm_isolate() == NULL)) {
// No isolate.
return false;
}
return isolate != Dart::vm_isolate();
}
void Profiler::SampleAllocation(Thread* thread,
intptr_t cid,
uint32_t identity_hash) {
ASSERT(thread != NULL);
OSThread* os_thread = thread->os_thread();
ASSERT(os_thread != NULL);
Isolate* isolate = thread->isolate();
if (!CheckIsolate(isolate)) {
return;
}
const bool exited_dart_code = thread->HasExitedDartCode();
SampleBlockBuffer* buffer = Profiler::sample_block_buffer();
if (buffer == nullptr) {
// Profiler not initialized.
return;
}
uintptr_t sp = OSThread::GetCurrentStackPointer();
uintptr_t fp = 0;
uintptr_t pc = OS::GetProgramCounter();
uintptr_t lr = 0;
COPY_FP_REGISTER(fp);
uword stack_lower = 0;
uword stack_upper = 0;
if (!InitialRegisterCheck(pc, fp, sp)) {
return;
}
if (!GetAndValidateThreadStackBounds(os_thread, thread, fp, sp, &stack_lower,
&stack_upper)) {
// Could not get stack boundary.
return;
}
Sample* sample =
SetupSample(thread, /*allocation_block*/ true, os_thread->trace_id());
if (sample == nullptr) {
// We were unable to assign a sample for this allocation.
counters_.sample_allocation_failure++;
return;
}
sample->SetAllocationCid(cid);
sample->set_allocation_identity_hash(identity_hash);
if (FLAG_profile_vm_allocation) {
ProfilerNativeStackWalker native_stack_walker(
&counters_, (isolate != NULL) ? isolate->main_port() : ILLEGAL_PORT,
sample, isolate->current_allocation_sample_block(), stack_lower,
stack_upper, pc, fp, sp);
native_stack_walker.walk();
} else if (exited_dart_code) {
ProfilerDartStackWalker dart_exit_stack_walker(
thread, sample, isolate->current_allocation_sample_block(), pc, fp, sp,
lr, /* allocation_sample*/ true);
dart_exit_stack_walker.walk();
} else {
// Fall back.
uintptr_t pc = OS::GetProgramCounter();
sample->SetAt(0, pc);
}
}
Sample* Profiler::SampleNativeAllocation(intptr_t skip_count,
uword address,
uintptr_t allocation_size) {
AllocationSampleBuffer* sample_buffer = Profiler::allocation_sample_buffer();
if (sample_buffer == NULL) {
return NULL;
}
uintptr_t sp = OSThread::GetCurrentStackPointer();
uintptr_t fp = 0;
uintptr_t pc = OS::GetProgramCounter();
COPY_FP_REGISTER(fp);
uword stack_lower = 0;
uword stack_upper = 0;
if (!InitialRegisterCheck(pc, fp, sp)) {
counters_.failure_native_allocation_sample.fetch_add(1);
return NULL;
}
if (!(OSThread::GetCurrentStackBounds(&stack_lower, &stack_upper) &&
ValidateThreadStackBounds(fp, sp, stack_lower, stack_upper))) {
// Could not get stack boundary.
counters_.failure_native_allocation_sample.fetch_add(1);
return NULL;
}
OSThread* os_thread = OSThread::Current();
Sample* sample = SetupSampleNative(sample_buffer, os_thread->trace_id());
if (sample == NULL) {
OS::PrintErr(
"Native memory profile sample buffer is full because there are more "
"than %" Pd
" outstanding allocations. Not recording allocation "
"0x%" Px " with size: %" Pu " bytes.\n",
sample_buffer->capacity(), address, allocation_size);
return NULL;
}
sample->set_native_allocation_address(address);
sample->set_native_allocation_size_bytes(allocation_size);
ProfilerNativeStackWalker native_stack_walker(
&counters_, ILLEGAL_PORT, sample, sample_buffer, stack_lower, stack_upper,
pc, fp, sp, skip_count);
native_stack_walker.walk();
return sample;
}
void Profiler::SampleThreadSingleFrame(Thread* thread,
Sample* sample,
uintptr_t pc) {
ASSERT(thread != NULL);
OSThread* os_thread = thread->os_thread();
ASSERT(os_thread != NULL);
Isolate* isolate = thread->isolate();
ASSERT(Profiler::sample_block_buffer() != nullptr);
// Increment counter for vm tag.
VMTagCounters* counters = isolate->vm_tag_counters();
ASSERT(counters != NULL);
if (thread->IsMutatorThread()) {
counters->Increment(sample->vm_tag());
}
// Write the single pc value.
sample->SetAt(0, pc);
}
void Profiler::SampleThread(Thread* thread,
const InterruptedThreadState& state) {
ASSERT(thread != NULL);
OSThread* os_thread = thread->os_thread();
ASSERT(os_thread != NULL);
Isolate* isolate = thread->isolate();
// Thread is not doing VM work.
if (thread->task_kind() == Thread::kUnknownTask) {
counters_.bail_out_unknown_task.fetch_add(1);
return;
}
if (StubCode::HasBeenInitialized() && StubCode::InJumpToFrameStub(state.pc)) {
// The JumpToFrame stub manually adjusts the stack pointer, frame
// pointer, and some isolate state. It is not safe to walk the
// stack when executing this stub.
counters_.bail_out_jump_to_exception_handler.fetch_add(1);
return;
}
const bool in_dart_code = thread->IsExecutingDartCode();
uintptr_t sp = 0;
uintptr_t fp = state.fp;
uintptr_t pc = state.pc;
uintptr_t lr = state.lr;
#if defined(USING_SIMULATOR)
Simulator* simulator = NULL;
#endif
if (in_dart_code) {
// If we're in Dart code, use the Dart stack pointer.
#if defined(USING_SIMULATOR)
simulator = isolate->simulator();
sp = simulator->get_register(SPREG);
fp = simulator->get_register(FPREG);
pc = simulator->get_pc();
lr = simulator->get_lr();
#else
sp = state.dsp;
#endif
} else {
// If we're in runtime code, use the C stack pointer.
sp = state.csp;
}
if (!CheckIsolate(isolate)) {
counters_.bail_out_check_isolate.fetch_add(1);
return;
}
SampleBlockBuffer* sample_block_buffer = Profiler::sample_block_buffer();
if (sample_block_buffer == nullptr) {
// Profiler not initialized.
return;
}
// Setup sample.
Sample* sample =
SetupSample(thread, /*allocation_block*/ false, os_thread->trace_id());
if (sample == nullptr) {
// We were unable to assign a sample for this profiler tick.
counters_.sample_allocation_failure++;
return;
}
if (thread->IsMutatorThread()) {
if (isolate->IsDeoptimizing()) {
counters_.single_frame_sample_deoptimizing.fetch_add(1);
SampleThreadSingleFrame(thread, sample, pc);
return;
}
if (isolate->group()->compaction_in_progress()) {
// The Dart stack isn't fully walkable.
SampleThreadSingleFrame(thread, sample, pc);
return;
}
}
if (!InitialRegisterCheck(pc, fp, sp)) {
counters_.single_frame_sample_register_check.fetch_add(1);
SampleThreadSingleFrame(thread, sample, pc);
return;
}
uword stack_lower = 0;
uword stack_upper = 0;
if (!GetAndValidateThreadStackBounds(os_thread, thread, fp, sp, &stack_lower,
&stack_upper)) {
counters_.single_frame_sample_get_and_validate_stack_bounds.fetch_add(1);
// Could not get stack boundary.
SampleThreadSingleFrame(thread, sample, pc);
return;
}
// At this point we have a valid stack boundary for this isolate and
// know that our initial stack and frame pointers are within the boundary.
// Increment counter for vm tag.
VMTagCounters* counters = isolate->vm_tag_counters();
ASSERT(counters != NULL);
if (thread->IsMutatorThread()) {
counters->Increment(sample->vm_tag());
}
ProfilerNativeStackWalker native_stack_walker(
&counters_, (isolate != NULL) ? isolate->main_port() : ILLEGAL_PORT,
sample, isolate->current_sample_block(), stack_lower, stack_upper, pc, fp,
sp);
const bool exited_dart_code = thread->HasExitedDartCode();
ProfilerDartStackWalker dart_stack_walker(
thread, sample, isolate->current_sample_block(), pc, fp, sp, lr,
/* allocation_sample*/ false);
// All memory access is done inside CollectSample.
CollectSample(isolate, exited_dart_code, in_dart_code, sample,
&native_stack_walker, &dart_stack_walker, pc, fp, sp,
&counters_);
}
CodeDescriptor::CodeDescriptor(const AbstractCode code) : code_(code) {}
uword CodeDescriptor::Start() const {
return code_.PayloadStart();
}
uword CodeDescriptor::Size() const {
return code_.Size();
}
int64_t CodeDescriptor::CompileTimestamp() const {
return code_.compile_timestamp();
}
CodeLookupTable::CodeLookupTable(Thread* thread) {
Build(thread);
}
class CodeLookupTableBuilder : public ObjectVisitor {
public:
explicit CodeLookupTableBuilder(CodeLookupTable* table) : table_(table) {
ASSERT(table_ != NULL);
}
~CodeLookupTableBuilder() {}
void VisitObject(ObjectPtr raw_obj) {
if (raw_obj->IsCode() && !Code::IsUnknownDartCode(Code::RawCast(raw_obj))) {
table_->Add(Code::Handle(Code::RawCast(raw_obj)));
}
}
private:
CodeLookupTable* table_;
};
void CodeLookupTable::Build(Thread* thread) {
ASSERT(thread != NULL);
Isolate* isolate = thread->isolate();
ASSERT(isolate != NULL);
Isolate* vm_isolate = Dart::vm_isolate();
ASSERT(vm_isolate != NULL);
// Clear.
code_objects_.Clear();
thread->CheckForSafepoint();
// Add all found Code objects.
{
TimelineBeginEndScope tl(Timeline::GetIsolateStream(),
"CodeLookupTable::Build HeapIterationScope");
HeapIterationScope iteration(thread);
CodeLookupTableBuilder cltb(this);
iteration.IterateVMIsolateObjects(&cltb);
iteration.IterateOldObjects(&cltb);
}
thread->CheckForSafepoint();
// Sort by entry.
code_objects_.Sort(CodeDescriptor::Compare);
#if defined(DEBUG)
if (length() <= 1) {
return;
}
ASSERT(FindCode(0) == NULL);
ASSERT(FindCode(~0) == NULL);
// Sanity check that we don't have duplicate entries and that the entries
// are sorted.
for (intptr_t i = 0; i < length() - 1; i++) {
const CodeDescriptor* a = At(i);
const CodeDescriptor* b = At(i + 1);
ASSERT(a->Start() < b->Start());
ASSERT(FindCode(a->Start()) == a);
ASSERT(FindCode(b->Start()) == b);
ASSERT(FindCode(a->Start() + a->Size() - 1) == a);
ASSERT(FindCode(b->Start() + b->Size() - 1) == b);
}
#endif
}
void CodeLookupTable::Add(const Object& code) {
ASSERT(!code.IsNull());
ASSERT(code.IsCode());
CodeDescriptor* cd = new CodeDescriptor(AbstractCode(code.ptr()));
code_objects_.Add(cd);
}
const CodeDescriptor* CodeLookupTable::FindCode(uword pc) const {
intptr_t first = 0;
intptr_t count = length();
while (count > 0) {
intptr_t current = first;
intptr_t step = count / 2;
current += step;
const CodeDescriptor* cd = At(current);
if (pc >= cd->Start()) {
first = ++current;
count -= step + 1;
} else {
count = step;
}
}
// First points to the first code object whose entry is greater than PC.
// That means the code object we need to check is first - 1.
if (first == 0) {
return NULL;
}
first--;
ASSERT(first >= 0);
ASSERT(first < length());
const CodeDescriptor* cd = At(first);
if (cd->Contains(pc)) {
return cd;
}
return NULL;
}
ProcessedSampleBuffer* SampleBuffer::BuildProcessedSampleBuffer(
SampleFilter* filter,
ProcessedSampleBuffer* buffer) {
Thread* thread = Thread::Current();
Zone* zone = thread->zone();
if (buffer == nullptr) {
buffer = new (zone) ProcessedSampleBuffer();
}
const intptr_t length = capacity();
for (intptr_t i = 0; i < length; i++) {
thread->CheckForSafepoint();
Sample* sample = At(i);
if (sample->ignore_sample()) {
// Bad sample.
continue;
}
if (!sample->head_sample()) {
// An inner sample in a chain of samples.
continue;
}
if (sample->timestamp() == 0) {
// Empty.
continue;
}
if (sample->At(0) == 0) {
// No frames.
continue;
}
if (filter != nullptr) {
// If we're requesting all the native allocation samples, we don't care
// whether or not we're in the same isolate as the sample.
if (sample->port() != filter->port()) {
// Another isolate.
continue;
}
if (!filter->TimeFilterSample(sample)) {
// Did not pass time filter.
continue;
}
if (!filter->TaskFilterSample(sample)) {
// Did not pass task filter.
continue;
}
if (!filter->FilterSample(sample)) {
// Did not pass filter.
continue;
}
}
buffer->Add(BuildProcessedSample(sample, buffer->code_lookup_table()));
}
return buffer;
}
ProcessedSample* SampleBuffer::BuildProcessedSample(
Sample* sample,
const CodeLookupTable& clt) {
Thread* thread = Thread::Current();
Zone* zone = thread->zone();
ProcessedSample* processed_sample = new (zone) ProcessedSample();
// Copy state bits from sample.
processed_sample->set_native_allocation_size_bytes(
sample->native_allocation_size_bytes());
processed_sample->set_timestamp(sample->timestamp());
processed_sample->set_tid(sample->tid());
processed_sample->set_vm_tag(sample->vm_tag());
processed_sample->set_user_tag(sample->user_tag());
if (sample->is_allocation_sample()) {
processed_sample->set_allocation_cid(sample->allocation_cid());
processed_sample->set_allocation_identity_hash(
sample->allocation_identity_hash());
}
processed_sample->set_first_frame_executing(!sample->exit_frame_sample());
// Copy stack trace from sample(s).
bool truncated = false;
Sample* current = sample;
while (current != NULL) {
for (intptr_t i = 0; i < Sample::kPCArraySizeInWords; i++) {
if (current->At(i) == 0) {
break;
}
processed_sample->Add(current->At(i));
}
truncated = truncated || current->truncated_trace();
current = Next(current);
}
if (!sample->exit_frame_sample()) {
processed_sample->FixupCaller(clt, /* pc_marker */ 0,
sample->GetStackBuffer());
}
processed_sample->set_truncated(truncated);
return processed_sample;
}
Sample* SampleBuffer::Next(Sample* sample) {
if (!sample->is_continuation_sample()) return NULL;
Sample* next_sample = sample->continuation_sample();
// Sanity check.
ASSERT(sample != next_sample);
// Detect invalid chaining.
if (sample->port() != next_sample->port()) {
return NULL;
}
if (sample->timestamp() != next_sample->timestamp()) {
return NULL;
}
if (sample->tid() != next_sample->tid()) {
return NULL;
}
return next_sample;
}
ProcessedSample::ProcessedSample()
: pcs_(Sample::kPCArraySizeInWords),
timestamp_(0),
vm_tag_(0),
user_tag_(0),
allocation_cid_(-1),
allocation_identity_hash_(0),
truncated_(false),
timeline_code_trie_(nullptr),
timeline_function_trie_(nullptr) {}
void ProcessedSample::FixupCaller(const CodeLookupTable& clt,
uword pc_marker,
uword* stack_buffer) {
const CodeDescriptor* cd = clt.FindCode(At(0));
if (cd == NULL) {
// No Dart code.
return;
}
if (cd->CompileTimestamp() > timestamp()) {
// Code compiled after sample. Ignore.
return;
}
CheckForMissingDartFrame(clt, cd, pc_marker, stack_buffer);
}
void ProcessedSample::CheckForMissingDartFrame(const CodeLookupTable& clt,
const CodeDescriptor* cd,
uword pc_marker,
uword* stack_buffer) {
ASSERT(cd != NULL);
const Code& code = Code::Handle(Code::RawCast(cd->code().ptr()));
ASSERT(!code.IsNull());
// Some stubs (and intrinsics) do not push a frame onto the stack leaving
// the frame pointer in the caller.
//
// PC -> STUB
// FP -> DART3 <-+
// DART2 <-| <- TOP FRAME RETURN ADDRESS.
// DART1 <-|
// .....
//
// In this case, traversing the linked stack frames will not collect a PC
// inside DART3. The stack will incorrectly be: STUB, DART2, DART1.
// In Dart code, after pushing the FP onto the stack, an IP in the current
// function is pushed onto the stack as well. This stack slot is called
// the PC marker. We can use the PC marker to insert DART3 into the stack
// so that it will correctly be: STUB, DART3, DART2, DART1. Note the
// inserted PC may not accurately reflect the true return address into DART3.
// The pc marker is our current best guess of a return address.
uword return_address = pc_marker;
// Attempt to find a better return address.
ReturnAddressLocator ral(At(0), stack_buffer, code);
if (!ral.LocateReturnAddress(&return_address)) {
ASSERT(return_address == pc_marker);
if (code.GetPrologueOffset() == 0) {
// Code has the prologue at offset 0. The frame is already setup and
// can be trusted.
return;
}
// Could not find a better return address than the pc_marker.
if (code.ContainsInstructionAt(return_address)) {
// PC marker is in the same code as pc, no missing frame.
return;
}
}
if (clt.FindCode(return_address) == NULL) {
// Return address is not from a Dart code object. Do not insert.
return;
}
if (return_address != 0) {
InsertAt(1, return_address);
}
}
ProcessedSampleBuffer::ProcessedSampleBuffer()
: code_lookup_table_(new CodeLookupTable(Thread::Current())) {
ASSERT(code_lookup_table_ != NULL);
}
void SampleBlockProcessor::Init() {
ASSERT(!initialized_);
if (monitor_ == nullptr) {
monitor_ = new Monitor();
}
ASSERT(monitor_ != nullptr);
initialized_ = true;
shutdown_ = false;
}
void SampleBlockProcessor::Startup() {
ASSERT(initialized_);
ASSERT(processor_thread_id_ == OSThread::kInvalidThreadJoinId);
MonitorLocker startup_ml(monitor_);
OSThread::Start("Dart Profiler SampleBlockProcessor", ThreadMain, 0);
while (!thread_running_) {
startup_ml.Wait();
}
ASSERT(processor_thread_id_ != OSThread::kInvalidThreadJoinId);
}
void SampleBlockProcessor::Cleanup() {
{
MonitorLocker shutdown_ml(monitor_);
if (shutdown_) {
// Already shutdown.
return;
}
shutdown_ = true;
// Notify.
shutdown_ml.Notify();
ASSERT(initialized_);
}
// Join the thread.
ASSERT(processor_thread_id_ != OSThread::kInvalidThreadJoinId);
OSThread::Join(processor_thread_id_);
processor_thread_id_ = OSThread::kInvalidThreadJoinId;
initialized_ = false;
ASSERT(!thread_running_);
}
class SampleBlockProcessorVisitor : public IsolateVisitor {
public:
SampleBlockProcessorVisitor() = default;
virtual ~SampleBlockProcessorVisitor() = default;
void VisitIsolate(Isolate* isolate) {
if (!isolate->should_process_blocks()) {
return;
}
Thread::EnterIsolateAsHelper(isolate, Thread::kSampleBlockTask);
Thread* thread = Thread::Current();
{
DisableThreadInterruptsScope dtis(thread);
isolate->ProcessFreeSampleBlocks(thread);
}
Thread::ExitIsolateAsHelper();
}
};
void SampleBlockProcessor::ThreadMain(uword parameters) {
ASSERT(initialized_);
{
// Signal to main thread we are ready.
MonitorLocker startup_ml(monitor_);
OSThread* os_thread = OSThread::Current();
ASSERT(os_thread != NULL);
processor_thread_id_ = OSThread::GetCurrentThreadJoinId(os_thread);
thread_running_ = true;
startup_ml.Notify();
}
SampleBlockProcessorVisitor visitor;
MonitorLocker wait_ml(monitor_);
// Wakeup every 100ms.
const int64_t wakeup_interval = 1000 * 100;
while (true) {
wait_ml.WaitMicros(wakeup_interval);
if (shutdown_) {
break;
}
Isolate::VisitIsolates(&visitor);
}
// Signal to main thread we are exiting.
thread_running_ = false;
}
#endif // !PRODUCT
} // namespace dart