[vm, compiler] Specialize unoptimized monomorphic and megamorphic calls.
dart-bytecode, arm64: +4.742% geomean
dart-bytecode-jit-unopt, arm64: +12.73% geomean
dart2js-compile, x64: +3.635% geomean
In the polymorphic and unlinked cases, call to a stub the does a linear scan against an ICData.
In the monomorphic case, call to a prologue of the expected target function that checks the expected receiver class. There is additional indirection in the JIT version compared to the AOT version to also tick a usage counter so the inliner can make good decisions.
In the megamorphic case, call to a stub that does a hash table lookup against a MegamorphicCache.
Megamorphic call sites face a loss of precision in usage counts. The call site count is not recorded and the usage counter of the target function is used as an approximation.
Monomorphic and megamorphic calls sites are reset to the polymorphic/unlinked state on hot reload.
Monomorphic and megamorphic calls sites do not check the stepping state, so they are reset to the polymorphic/unlinked state when stepping begins and disabled.
Back-edges now increment the usage counter in addition to checking it. This ensures function with loops containing monomorphic calls will eventually cross the optimization threshold.
Fixed backwards use of kMonomorphicEntryOffset and kPolymorphicEntryOffset.
Fixed C stack overflow when bouncing between the KBC interpreter and a simulator.
Bug: https://github.com/dart-lang/sdk/issues/26780
Bug: https://github.com/dart-lang/sdk/issues/36409
Bug: https://github.com/dart-lang/sdk/issues/36731
Change-Id: I78a49cccd962703a459288e71ce246ed845df474
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/102820
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status
index 5b8bc01..5d70c9e 100644
--- a/runtime/tests/vm/vm.status
+++ b/runtime/tests/vm/vm.status
@@ -13,8 +13,9 @@
cc/IsolateReload_PendingUnqualifiedCall_StaticToInstance: Fail # Issue 32981
cc/IsolateReload_RunNewFieldInitializersWithGenerics: Fail # Issue 32299
dart/data_uri_import_test/none: SkipByDesign
-dart/snapshot_version_test: Skip # This test is a Dart1 test (script snapshot)
+dart/entrypoints/jit: Skip # Tests with brittle dependencies on usage counters - Issue 37144
dart/slow_path_shared_stub_test: Pass, Slow # Uses --shared-slow-path-triggers-gc flag.
+dart/snapshot_version_test: Skip # This test is a Dart1 test (script snapshot)
dart/stack_overflow_shared_test: Pass, Slow # Uses --shared-slow-path-triggers-gc flag.
dart/use_bare_instructions_flag_test: Pass, Slow # Spawns several subprocesses
diff --git a/runtime/vm/code_patcher.h b/runtime/vm/code_patcher.h
index 239a69d..ceacea6 100644
--- a/runtime/vm/code_patcher.h
+++ b/runtime/vm/code_patcher.h
@@ -49,12 +49,18 @@
// in given code.
static RawCode* GetStaticCallTargetAt(uword return_address, const Code& code);
- // Get instance call information. Returns the call target and sets each
- // of the output parameters ic_data and arguments_descriptor if they are
- // non-NULL.
+ // Get instance call information. Returns the call target and sets the output
+ // parameter data if non-NULL.
static RawCode* GetInstanceCallAt(uword return_address,
- const Code& code,
- ICData* ic_data);
+ const Code& caller_code,
+ Object* data);
+
+ // Change the state of an instance call by patching the corresponding object
+ // pool entries (non-IA32) or instructions (IA32).
+ static void PatchInstanceCallAt(uword return_address,
+ const Code& caller_code,
+ const Object& data,
+ const Code& target);
// Return target of an unoptimized static call and its ICData object
// (calls target via a stub).
@@ -79,22 +85,22 @@
#if defined(TARGET_ARCH_DBC)
static NativeFunctionWrapper GetNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction* target);
#else
static RawCode* GetNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction* target);
#endif
#if defined(TARGET_ARCH_DBC)
static void PatchNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction target,
NativeFunctionWrapper trampoline);
#else
static void PatchNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction target,
const Code& trampoline);
#endif
diff --git a/runtime/vm/code_patcher_arm.cc b/runtime/vm/code_patcher_arm.cc
index 362b8b0..024a6ab 100644
--- a/runtime/vm/code_patcher_arm.cc
+++ b/runtime/vm/code_patcher_arm.cc
@@ -33,23 +33,33 @@
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
- const Code& code,
- ICData* ic_data) {
- ASSERT(code.ContainsInstructionAt(return_address));
- CallPattern call(return_address, code);
- if (ic_data != NULL) {
- *ic_data = call.IcData();
+ const Code& caller_code,
+ Object* data) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ ICCallPattern call(return_address, caller_code);
+ if (data != NULL) {
+ *data = call.Data();
}
return call.TargetCode();
}
+void CodePatcher::PatchInstanceCallAt(uword return_address,
+ const Code& caller_code,
+ const Object& data,
+ const Code& target) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ ICCallPattern call(return_address, caller_code);
+ call.SetData(data);
+ call.SetTargetCode(target);
+}
+
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
ICData* ic_data_result) {
- ASSERT(code.ContainsInstructionAt(return_address));
- CallPattern static_call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ ICCallPattern static_call(return_address, caller_code);
ICData& ic_data = ICData::Handle();
- ic_data = static_call.IcData();
+ ic_data ^= static_call.Data();
if (ic_data_result != NULL) {
*ic_data_result = ic_data.raw();
}
diff --git a/runtime/vm/code_patcher_arm64.cc b/runtime/vm/code_patcher_arm64.cc
index 7716090..ce3c4dc 100644
--- a/runtime/vm/code_patcher_arm64.cc
+++ b/runtime/vm/code_patcher_arm64.cc
@@ -68,23 +68,33 @@
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
- const Code& code,
- ICData* ic_data) {
- ASSERT(code.ContainsInstructionAt(return_address));
- CallPattern call(return_address, code);
- if (ic_data != NULL) {
- *ic_data = call.IcData();
+ const Code& caller_code,
+ Object* data) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ ICCallPattern call(return_address, caller_code);
+ if (data != NULL) {
+ *data = call.Data();
}
return call.TargetCode();
}
+void CodePatcher::PatchInstanceCallAt(uword return_address,
+ const Code& caller_code,
+ const Object& data,
+ const Code& target) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ ICCallPattern call(return_address, caller_code);
+ call.SetData(data);
+ call.SetTargetCode(target);
+}
+
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
const Code& code,
ICData* ic_data_result) {
ASSERT(code.ContainsInstructionAt(return_address));
- CallPattern static_call(return_address, code);
+ ICCallPattern static_call(return_address, code);
ICData& ic_data = ICData::Handle();
- ic_data ^= static_call.IcData();
+ ic_data ^= static_call.Data();
if (ic_data_result != NULL) {
*ic_data_result = ic_data.raw();
}
@@ -132,20 +142,20 @@
}
void CodePatcher::PatchNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction target,
const Code& trampoline) {
- ASSERT(code.ContainsInstructionAt(return_address));
- NativeCallPattern call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ NativeCallPattern call(return_address, caller_code);
call.set_target(trampoline);
call.set_native_function(target);
}
RawCode* CodePatcher::GetNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction* target) {
- ASSERT(code.ContainsInstructionAt(return_address));
- NativeCallPattern call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ NativeCallPattern call(return_address, caller_code);
*target = call.native_function();
return call.target();
}
diff --git a/runtime/vm/code_patcher_arm64_test.cc b/runtime/vm/code_patcher_arm64_test.cc
index c2437cf..a31c467 100644
--- a/runtime/vm/code_patcher_arm64_test.cc
+++ b/runtime/vm/code_patcher_arm64_test.cc
@@ -39,12 +39,22 @@
ArgumentsDescriptor::New(kTypeArgsLen, kNumArgs, Object::null_array()));
const ICData& ic_data = ICData::ZoneHandle(ICData::New(
function, target_name, args_descriptor, 15, 1, ICData::kInstance));
+ const Code& stub = StubCode::OneArgCheckInlineCache();
// Code accessing pp is generated, but not executed. Uninitialized pp is OK.
__ set_constant_pool_allowed(true);
- __ LoadObject(R5, ic_data);
- __ BranchLinkPatchable(StubCode::OneArgCheckInlineCache());
+ ObjectPoolBuilder& op = __ object_pool_builder();
+ const intptr_t ic_data_index =
+ op.AddObject(ic_data, ObjectPool::Patchability::kPatchable);
+ const intptr_t stub_index =
+ op.AddObject(stub, ObjectPool::Patchability::kPatchable);
+ ASSERT((ic_data_index + 1) == stub_index);
+ __ LoadDoubleWordFromPoolOffset(R5, CODE_REG,
+ ObjectPool::element_offset(ic_data_index));
+ __ ldr(LR, FieldAddress(CODE_REG, Code::entry_point_offset(
+ Code::EntryKind::kMonomorphic)));
+ __ blr(LR);
__ ret();
}
diff --git a/runtime/vm/code_patcher_dbc.cc b/runtime/vm/code_patcher_dbc.cc
index 9c478ff..93bc87a 100644
--- a/runtime/vm/code_patcher_dbc.cc
+++ b/runtime/vm/code_patcher_dbc.cc
@@ -33,23 +33,33 @@
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
- const Code& code,
- ICData* ic_data) {
- ASSERT(code.ContainsInstructionAt(return_address));
- CallPattern call(return_address, code);
- if (ic_data != NULL) {
- *ic_data = call.IcData();
+ const Code& caller_code,
+ Object* cache) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ CallPattern call(return_address, caller_code);
+ if (cache != NULL) {
+ *cache = call.Data();
}
return call.TargetCode();
}
+void CodePatcher::PatchInstanceCallAt(uword return_address,
+ const Code& caller_code,
+ const Object& data,
+ const Code& target) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ CallPattern call(return_address, caller_code);
+ call.SetData(data);
+ call.SetTargetCode(target);
+}
+
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
ICData* ic_data_result) {
- ASSERT(code.ContainsInstructionAt(return_address));
- CallPattern static_call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ CallPattern static_call(return_address, caller_code);
ICData& ic_data = ICData::Handle();
- ic_data ^= static_call.IcData();
+ ic_data ^= static_call.Data();
if (ic_data_result != NULL) {
*ic_data_result = ic_data.raw();
}
@@ -91,10 +101,10 @@
}
NativeFunctionWrapper CodePatcher::GetNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction* target) {
- ASSERT(code.ContainsInstructionAt(return_address));
- NativeCallPattern call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ NativeCallPattern call(return_address, caller_code);
*target = call.native_function();
return call.target();
}
diff --git a/runtime/vm/code_patcher_ia32.cc b/runtime/vm/code_patcher_ia32.cc
index ccc8c07..81fa738 100644
--- a/runtime/vm/code_patcher_ia32.cc
+++ b/runtime/vm/code_patcher_ia32.cc
@@ -72,17 +72,40 @@
DISALLOW_IMPLICIT_CONSTRUCTORS(NativeCall);
};
+// b9xxxxxxxx mov ecx,<data>
+// bfyyyyyyyy mov edi,<target>
+// ff5707 call [edi+<monomorphic-entry-offset>]
class InstanceCall : public UnoptimizedCall {
public:
explicit InstanceCall(uword return_address)
: UnoptimizedCall(return_address) {
#if defined(DEBUG)
- ICData& test_ic_data = ICData::Handle();
- test_ic_data ^= ic_data();
- ASSERT(test_ic_data.NumArgsTested() > 0);
+ Object& test_data = Object::Handle(data());
+ ASSERT(test_data.IsArray() || test_data.IsICData() ||
+ test_data.IsMegamorphicCache());
+ if (test_data.IsICData()) {
+ ASSERT(ICData::Cast(test_data).NumArgsTested() > 0);
+ }
#endif // DEBUG
}
+ RawObject* data() const { return *reinterpret_cast<RawObject**>(start_ + 1); }
+ void set_data(const Object& data) const {
+ uword* cache_addr = reinterpret_cast<uword*>(start_ + 1);
+ uword imm = reinterpret_cast<uword>(data.raw());
+ *cache_addr = imm;
+ }
+
+ RawCode* target() const {
+ const uword imm = *reinterpret_cast<uword*>(start_ + 6);
+ return reinterpret_cast<RawCode*>(imm);
+ }
+ void set_target(const Code& target) const {
+ uword* target_addr = reinterpret_cast<uword*>(start_ + 6);
+ uword imm = reinterpret_cast<uword>(target.raw());
+ *target_addr = imm;
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(InstanceCall);
};
@@ -168,20 +191,32 @@
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
- const Code& code,
- ICData* ic_data) {
- ASSERT(code.ContainsInstructionAt(return_address));
+ const Code& caller_code,
+ Object* data) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
InstanceCall call(return_address);
- if (ic_data != NULL) {
- *ic_data ^= call.ic_data();
+ if (data != NULL) {
+ *data = call.data();
}
- return Code::null();
+ return call.target();
+}
+
+void CodePatcher::PatchInstanceCallAt(uword return_address,
+ const Code& caller_code,
+ const Object& data,
+ const Code& target) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ const Instructions& instrs = Instructions::Handle(caller_code.instructions());
+ WritableInstructionsScope writable(instrs.PayloadStart(), instrs.Size());
+ InstanceCall call(return_address);
+ call.set_data(data);
+ call.set_target(target);
}
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
ICData* ic_data_result) {
- ASSERT(code.ContainsInstructionAt(return_address));
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
UnoptimizedStaticCall static_call(return_address);
ICData& ic_data = ICData::Handle();
ic_data ^= static_call.ic_data();
@@ -214,14 +249,14 @@
}
void CodePatcher::PatchNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction target,
const Code& trampoline) {
UNREACHABLE();
}
RawCode* CodePatcher::GetNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction* target) {
UNREACHABLE();
return NULL;
diff --git a/runtime/vm/code_patcher_x64.cc b/runtime/vm/code_patcher_x64.cc
index f53cf2a..e67d187 100644
--- a/runtime/vm/code_patcher_x64.cc
+++ b/runtime/vm/code_patcher_x64.cc
@@ -77,8 +77,6 @@
intptr_t argument_index() const { return argument_index_; }
- RawObject* ic_data() const { return object_pool_.ObjectAt(argument_index()); }
-
RawCode* target() const {
Code& code = Code::Handle();
code ^= object_pool_.ObjectAt(code_index_);
@@ -123,20 +121,29 @@
InstanceCall(uword return_address, const Code& code)
: UnoptimizedCall(return_address, code) {
#if defined(DEBUG)
- ICData& test_ic_data = ICData::Handle();
- test_ic_data ^= ic_data();
- ASSERT(test_ic_data.NumArgsTested() > 0);
+ Object& test_data = Object::Handle(data());
+ ASSERT(test_data.IsArray() || test_data.IsICData() ||
+ test_data.IsMegamorphicCache());
+ if (test_data.IsICData()) {
+ ASSERT(ICData::Cast(test_data).NumArgsTested() > 0);
+ }
#endif // DEBUG
}
+ RawObject* data() const { return object_pool_.ObjectAt(argument_index()); }
+ void set_data(const Object& data) const {
+ ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
+ object_pool_.SetObjectAt(argument_index(), data);
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(InstanceCall);
};
class UnoptimizedStaticCall : public UnoptimizedCall {
public:
- UnoptimizedStaticCall(uword return_address, const Code& code)
- : UnoptimizedCall(return_address, code) {
+ UnoptimizedStaticCall(uword return_address, const Code& caller_code)
+ : UnoptimizedCall(return_address, caller_code) {
#if defined(DEBUG)
ICData& test_ic_data = ICData::Handle();
test_ic_data ^= ic_data();
@@ -144,6 +151,8 @@
#endif // DEBUG
}
+ RawObject* ic_data() const { return object_pool_.ObjectAt(argument_index()); }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedStaticCall);
};
@@ -152,8 +161,8 @@
// the object pool.
class PoolPointerCall : public ValueObject {
public:
- explicit PoolPointerCall(uword return_address, const Code& code)
- : object_pool_(ObjectPool::Handle(code.GetObjectPool())),
+ explicit PoolPointerCall(uword return_address, const Code& caller_code)
+ : object_pool_(ObjectPool::Handle(caller_code.GetObjectPool())),
code_index_(-1) {
uword pc = return_address;
@@ -424,25 +433,35 @@
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
- const Code& code,
- ICData* ic_data) {
- ASSERT(code.ContainsInstructionAt(return_address));
- InstanceCall call(return_address, code);
- if (ic_data != NULL) {
- *ic_data ^= call.ic_data();
+ const Code& caller_code,
+ Object* data) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ InstanceCall call(return_address, caller_code);
+ if (data != NULL) {
+ *data = call.data();
}
return call.target();
}
+void CodePatcher::PatchInstanceCallAt(uword return_address,
+ const Code& caller_code,
+ const Object& data,
+ const Code& target) {
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ InstanceCall call(return_address, caller_code);
+ call.set_data(data);
+ call.set_target(target);
+}
+
void CodePatcher::InsertDeoptimizationCallAt(uword start) {
UNREACHABLE();
}
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
ICData* ic_data_result) {
- ASSERT(code.ContainsInstructionAt(return_address));
- UnoptimizedStaticCall static_call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ UnoptimizedStaticCall static_call(return_address, caller_code);
ICData& ic_data = ICData::Handle();
ic_data ^= static_call.ic_data();
if (ic_data_result != NULL) {
@@ -492,20 +511,20 @@
}
void CodePatcher::PatchNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction target,
const Code& trampoline) {
- ASSERT(code.ContainsInstructionAt(return_address));
- NativeCall call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ NativeCall call(return_address, caller_code);
call.set_target(trampoline);
call.set_native_function(target);
}
RawCode* CodePatcher::GetNativeCallAt(uword return_address,
- const Code& code,
+ const Code& caller_code,
NativeFunction* target) {
- ASSERT(code.ContainsInstructionAt(return_address));
- NativeCall call(return_address, code);
+ ASSERT(caller_code.ContainsInstructionAt(return_address));
+ NativeCall call(return_address, caller_code);
*target = call.native_function();
return call.target();
}
diff --git a/runtime/vm/compiler/aot/aot_call_specializer.cc b/runtime/vm/compiler/aot/aot_call_specializer.cc
index b3c2ea3..196a9df 100644
--- a/runtime/vm/compiler/aot/aot_call_specializer.cc
+++ b/runtime/vm/compiler/aot/aot_call_specializer.cc
@@ -161,7 +161,8 @@
Function::Handle(Z, call->ResolveForReceiverClass(cls));
ASSERT(!function.IsNull());
const Function& target = Function::ZoneHandle(Z, function.raw());
- StaticCallInstr* static_call = StaticCallInstr::FromCall(Z, call, target);
+ StaticCallInstr* static_call =
+ StaticCallInstr::FromCall(Z, call, target, call->CallCount());
static_call->SetResultType(Z, CompileType::FromCid(kTypeCid));
call->ReplaceWith(static_call, current_iterator());
return true;
@@ -849,7 +850,8 @@
CallTargets* targets = CallTargets::Create(Z, unary_checks);
ASSERT(targets->HasSingleTarget());
const Function& target = targets->FirstTarget();
- StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
+ StaticCallInstr* call = StaticCallInstr::FromCall(
+ Z, instr, target, targets->AggregateCallCount());
instr->ReplaceWith(call, current_iterator());
return;
}
@@ -911,7 +913,8 @@
Function::Handle(Z, instr->ResolveForReceiverClass(receiver_class));
if (!function.IsNull()) {
const Function& target = Function::ZoneHandle(Z, function.raw());
- StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
+ StaticCallInstr* call =
+ StaticCallInstr::FromCall(Z, instr, target, instr->CallCount());
instr->ReplaceWith(call, current_iterator());
return;
}
@@ -1024,7 +1027,8 @@
// We have computed that there is only a single target for this call
// within the whole hierarchy. Replace InstanceCall with StaticCall.
const Function& target = Function::ZoneHandle(Z, single_target.raw());
- StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
+ StaticCallInstr* call =
+ StaticCallInstr::FromCall(Z, instr, target, instr->CallCount());
instr->ReplaceWith(call, current_iterator());
return;
} else if ((ic_data.raw() != ICData::null()) &&
@@ -1181,7 +1185,8 @@
Z, call->instance_call()->ResolveForReceiverClass(receiver_class));
if (!function.IsNull()) {
// Only one target. Replace by static call.
- StaticCallInstr* new_call = StaticCallInstr::FromCall(Z, call, function);
+ StaticCallInstr* new_call =
+ StaticCallInstr::FromCall(Z, call, function, call->CallCount());
call->ReplaceWith(new_call, current_iterator());
}
}
diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc
index fa570c5..1c06796 100644
--- a/runtime/vm/compiler/assembler/assembler_arm.cc
+++ b/runtime/vm/compiler/assembler/assembler_arm.cc
@@ -3393,9 +3393,9 @@
LeaveDartFrame();
}
-// R0 receiver, R9 guarded cid as Smi.
+// R0 receiver, R9 ICData entries array
// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
-void Assembler::MonomorphicCheckedEntry() {
+void Assembler::MonomorphicCheckedEntryJIT() {
has_single_entry_point_ = false;
#if defined(TESTING) || defined(DEBUG)
bool saved_use_far_branches = use_far_branches();
@@ -3404,19 +3404,65 @@
intptr_t start = CodeSize();
Comment("MonomorphicCheckedEntry");
- ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffset);
+ ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetJIT);
+
+ const intptr_t cid_offset = target::Array::element_offset(0);
+ const intptr_t count_offset = target::Array::element_offset(1);
+
+ // Sadly this cannot use ldm because ldm takes no offset.
+ ldr(R1, FieldAddress(R9, cid_offset));
+ ldr(R2, FieldAddress(R9, count_offset));
+ LoadClassIdMayBeSmi(IP, R0);
+ add(R2, R2, Operand(target::ToRawSmi(1)));
+ cmp(R1, Operand(IP, LSL, 1));
+ Branch(Address(THR, Thread::monomorphic_miss_entry_offset()), NE);
+ str(R2, FieldAddress(R9, count_offset));
+ LoadImmediate(R4, 0); // GC-safe for OptimizeInvokedFunction.
+
+ // Fall through to unchecked entry.
+ ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetJIT);
+
+#if defined(TESTING) || defined(DEBUG)
+ set_use_far_branches(saved_use_far_branches);
+#endif
+}
+
+// R0 receiver, R9 guarded cid as Smi.
+// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
+void Assembler::MonomorphicCheckedEntryAOT() {
+ has_single_entry_point_ = false;
+#if defined(TESTING) || defined(DEBUG)
+ bool saved_use_far_branches = use_far_branches();
+ set_use_far_branches(false);
+#endif
+ intptr_t start = CodeSize();
+
+ Comment("MonomorphicCheckedEntry");
+ ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetAOT);
+
LoadClassIdMayBeSmi(IP, R0);
cmp(R9, Operand(IP, LSL, 1));
Branch(Address(THR, Thread::monomorphic_miss_entry_offset()), NE);
// Fall through to unchecked entry.
- ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffset);
+ ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetAOT);
#if defined(TESTING) || defined(DEBUG)
set_use_far_branches(saved_use_far_branches);
#endif
}
+void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
+ has_single_entry_point_ = false;
+ while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
+ bkpt(0);
+ }
+ b(label);
+ while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
+ bkpt(0);
+ }
+}
+
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(Register stats_addr_reg, Label* trace) {
ASSERT(stats_addr_reg != kNoRegister);
diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h
index ceb1820..19ba44f 100644
--- a/runtime/vm/compiler/assembler/assembler_arm.h
+++ b/runtime/vm/compiler/assembler/assembler_arm.h
@@ -1058,7 +1058,9 @@
void EnterStubFrame();
void LeaveStubFrame();
- void MonomorphicCheckedEntry();
+ void MonomorphicCheckedEntryJIT();
+ void MonomorphicCheckedEntryAOT();
+ void BranchOnMonomorphicCheckedEntryJIT(Label* label);
// The register into which the allocation stats table is loaded with
// LoadAllocationStatsAddress should be passed to MaybeTraceAllocation and
diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc
index 5c1b902..d9a731c 100644
--- a/runtime/vm/compiler/assembler/assembler_arm64.cc
+++ b/runtime/vm/compiler/assembler/assembler_arm64.cc
@@ -1461,9 +1461,44 @@
LeaveDartFrame();
}
+// R0 receiver, R5 ICData entries array
+// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
+void Assembler::MonomorphicCheckedEntryJIT() {
+ ASSERT(has_single_entry_point_);
+ has_single_entry_point_ = false;
+ const bool saved_use_far_branches = use_far_branches();
+ set_use_far_branches(false);
+
+ Label immediate, miss;
+ Bind(&miss);
+ ldr(IP0, Address(THR, Thread::monomorphic_miss_entry_offset()));
+ br(IP0);
+
+ Comment("MonomorphicCheckedEntry");
+ ASSERT(CodeSize() == Instructions::kMonomorphicEntryOffsetJIT);
+
+ const intptr_t cid_offset = target::Array::element_offset(0);
+ const intptr_t count_offset = target::Array::element_offset(1);
+
+ // Sadly this cannot use ldp because ldp requires aligned offsets.
+ ldr(R1, FieldAddress(R5, cid_offset));
+ ldr(R2, FieldAddress(R5, count_offset));
+ LoadClassIdMayBeSmi(IP0, R0);
+ add(R2, R2, Operand(target::ToRawSmi(1)));
+ cmp(R1, Operand(IP0, LSL, 1));
+ b(&miss, NE);
+ str(R2, FieldAddress(R5, count_offset));
+ LoadImmediate(R4, 0); // GC-safe for OptimizeInvokedFunction.
+
+ // Fall through to unchecked entry.
+ ASSERT(CodeSize() == Instructions::kPolymorphicEntryOffsetJIT);
+
+ set_use_far_branches(saved_use_far_branches);
+}
+
// R0 receiver, R5 guarded cid as Smi.
// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
-void Assembler::MonomorphicCheckedEntry() {
+void Assembler::MonomorphicCheckedEntryAOT() {
ASSERT(has_single_entry_point_);
has_single_entry_point_ = false;
bool saved_use_far_branches = use_far_branches();
@@ -1477,17 +1512,28 @@
br(IP0);
Comment("MonomorphicCheckedEntry");
- ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffset);
+ ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetAOT);
LoadClassIdMayBeSmi(IP0, R0);
cmp(R5, Operand(IP0, LSL, 1));
b(&miss, NE);
// Fall through to unchecked entry.
- ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffset);
+ ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetAOT);
set_use_far_branches(saved_use_far_branches);
}
+void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
+ has_single_entry_point_ = false;
+ while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
+ brk(0);
+ }
+ b(label);
+ while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
+ brk(0);
+ }
+}
+
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
Register temp_reg,
diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h
index cb9af12..b8808f5 100644
--- a/runtime/vm/compiler/assembler/assembler_arm64.h
+++ b/runtime/vm/compiler/assembler/assembler_arm64.h
@@ -1561,7 +1561,9 @@
void EnterStubFrame();
void LeaveStubFrame();
- void MonomorphicCheckedEntry();
+ void MonomorphicCheckedEntryJIT();
+ void MonomorphicCheckedEntryAOT();
+ void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void UpdateAllocationStats(intptr_t cid);
diff --git a/runtime/vm/compiler/assembler/assembler_dbc.h b/runtime/vm/compiler/assembler/assembler_dbc.h
index b88795f..38a8920 100644
--- a/runtime/vm/compiler/assembler/assembler_dbc.h
+++ b/runtime/vm/compiler/assembler/assembler_dbc.h
@@ -39,7 +39,8 @@
// Misc. functionality
intptr_t prologue_offset() const { return 0; }
- void MonomorphicCheckedEntry() {}
+ void MonomorphicCheckedEntryJIT() {}
+ void MonomorphicCheckedEntryAOT() {}
// Debugging and bringup support.
void Stop(const char* message) override;
diff --git a/runtime/vm/compiler/assembler/assembler_ia32.cc b/runtime/vm/compiler/assembler/assembler_ia32.cc
index 7311298..73031ba 100644
--- a/runtime/vm/compiler/assembler/assembler_ia32.cc
+++ b/runtime/vm/compiler/assembler/assembler_ia32.cc
@@ -2097,6 +2097,56 @@
#endif
}
+// EBX receiver, ECX ICData entries array
+// Preserve EDX (ARGS_DESC_REG), not required today, but maybe later.
+void Assembler::MonomorphicCheckedEntryJIT() {
+ has_single_entry_point_ = false;
+ intptr_t start = CodeSize();
+ Label have_cid, miss;
+ Bind(&miss);
+ jmp(Address(THR, Thread::monomorphic_miss_entry_offset()));
+
+ Comment("MonomorphicCheckedEntry");
+ ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetJIT);
+
+ const intptr_t cid_offset = target::Array::element_offset(0);
+ const intptr_t count_offset = target::Array::element_offset(1);
+
+ movl(EAX, Immediate(kSmiCid << 1));
+ testl(EBX, Immediate(kSmiTagMask));
+ j(ZERO, &have_cid, kNearJump);
+ LoadClassId(EAX, EBX);
+ SmiTag(EAX);
+ Bind(&have_cid);
+ // EAX: cid as Smi
+
+ cmpl(EAX, FieldAddress(ECX, cid_offset));
+ j(NOT_EQUAL, &miss, Assembler::kNearJump);
+ addl(FieldAddress(ECX, count_offset), Immediate(target::ToRawSmi(1)));
+ xorl(EDX, EDX); // GC-safe for OptimizeInvokedFunction.
+ nop(1);
+
+ // Fall through to unchecked entry.
+ ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetJIT);
+}
+
+// EBX receiver, ECX guarded cid as Smi.
+// Preserve EDX (ARGS_DESC_REG), not required today, but maybe later.
+void Assembler::MonomorphicCheckedEntryAOT() {
+ UNIMPLEMENTED();
+}
+
+void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
+ has_single_entry_point_ = false;
+ while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
+ int3();
+ }
+ jmp(label);
+ while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
+ int3();
+ }
+}
+
void Assembler::TransitionGeneratedToNative(Register destination_address,
Register new_exit_frame,
Register scratch) {
diff --git a/runtime/vm/compiler/assembler/assembler_ia32.h b/runtime/vm/compiler/assembler/assembler_ia32.h
index ad6149b..4bc1e6f 100644
--- a/runtime/vm/compiler/assembler/assembler_ia32.h
+++ b/runtime/vm/compiler/assembler/assembler_ia32.h
@@ -646,7 +646,9 @@
void LeaveFrame();
void ReserveAlignedFrameSpace(intptr_t frame_space);
- void MonomorphicCheckedEntry() {}
+ void MonomorphicCheckedEntryJIT();
+ void MonomorphicCheckedEntryAOT();
+ void BranchOnMonomorphicCheckedEntryJIT(Label* label);
// In debug mode, this generates code to check that:
// FP + kExitLinkSlotFromEntryFp == SP
@@ -744,8 +746,6 @@
// Needs a temporary register.
void MoveMemoryToMemory(Address to, Address from, Register tmp);
- bool has_single_entry_point() const { return true; }
-
// Set up a Dart frame on entry with a frame pointer and PC information to
// enable easy access to the RawInstruction object of code corresponding
// to this frame.
diff --git a/runtime/vm/compiler/assembler/assembler_x64.cc b/runtime/vm/compiler/assembler/assembler_x64.cc
index 1a6596e..a90db6d 100644
--- a/runtime/vm/compiler/assembler/assembler_x64.cc
+++ b/runtime/vm/compiler/assembler/assembler_x64.cc
@@ -1742,9 +1742,9 @@
LeaveDartFrame();
}
-// RDX receiver, RBX guarded cid as Smi.
+// RDX receiver, RBX ICData entries array
// Preserve R10 (ARGS_DESC_REG), not required today, but maybe later.
-void Assembler::MonomorphicCheckedEntry() {
+void Assembler::MonomorphicCheckedEntryJIT() {
has_single_entry_point_ = false;
intptr_t start = CodeSize();
Label have_cid, miss;
@@ -1756,7 +1756,38 @@
nop(1);
Comment("MonomorphicCheckedEntry");
- ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffset);
+ ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetJIT);
+ ASSERT((CodeSize() & kSmiTagMask) == kSmiTag);
+
+ const intptr_t cid_offset = target::Array::element_offset(0);
+ const intptr_t count_offset = target::Array::element_offset(1);
+
+ LoadTaggedClassIdMayBeSmi(RAX, RDX);
+
+ cmpq(RAX, FieldAddress(RBX, cid_offset));
+ j(NOT_EQUAL, &miss, Assembler::kNearJump);
+ addl(FieldAddress(RBX, count_offset), Immediate(target::ToRawSmi(1)));
+ xorq(R10, R10); // GC-safe for OptimizeInvokedFunction.
+ nop(1);
+
+ // Fall through to unchecked entry.
+ ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetJIT);
+ ASSERT(((CodeSize() - start) & kSmiTagMask) == kSmiTag);
+}
+
+void Assembler::MonomorphicCheckedEntryAOT() {
+ has_single_entry_point_ = false;
+ intptr_t start = CodeSize();
+ Label have_cid, miss;
+ Bind(&miss);
+ jmp(Address(THR, Thread::monomorphic_miss_entry_offset()));
+
+ // Ensure the monomorphic entry is 2-byte aligned (so GC can see them if we
+ // store them in ICData / MegamorphicCache arrays)
+ nop(1);
+
+ Comment("MonomorphicCheckedEntry");
+ ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetAOT);
ASSERT((CodeSize() & kSmiTagMask) == kSmiTag);
movq(RAX, Immediate(kSmiCid));
@@ -1774,10 +1805,21 @@
nop(1);
// Fall through to unchecked entry.
- ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffset);
+ ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetAOT);
ASSERT(((CodeSize() - start) & kSmiTagMask) == kSmiTag);
}
+void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
+ has_single_entry_point_ = false;
+ while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
+ int3();
+ }
+ jmp(label);
+ while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
+ int3();
+ }
+}
+
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
Label* trace,
diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h
index 4df9250..73cadd0 100644
--- a/runtime/vm/compiler/assembler/assembler_x64.h
+++ b/runtime/vm/compiler/assembler/assembler_x64.h
@@ -885,7 +885,9 @@
void EnterStubFrame();
void LeaveStubFrame();
- void MonomorphicCheckedEntry();
+ void MonomorphicCheckedEntryJIT();
+ void MonomorphicCheckedEntryAOT();
+ void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void UpdateAllocationStats(intptr_t cid);
diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.cc b/runtime/vm/compiler/backend/flow_graph_compiler.cc
index 440b6e7d..2fc7029 100644
--- a/runtime/vm/compiler/backend/flow_graph_compiler.cc
+++ b/runtime/vm/compiler/backend/flow_graph_compiler.cc
@@ -339,7 +339,8 @@
// Intrinsification happened.
if (parsed_function().function().IsDynamicFunction()) {
- return Instructions::kMonomorphicEntryOffset;
+ return FLAG_precompiled_mode ? Instructions::kPolymorphicEntryOffsetAOT
+ : Instructions::kPolymorphicEntryOffsetJIT;
}
return 0;
@@ -1360,7 +1361,7 @@
if (FLAG_precompiled_mode) {
// TODO(#34162): Support unchecked entry-points in precompiled mode.
ic_data = ic_data.AsUnaryClassChecks();
- EmitSwitchableInstanceCall(ic_data, deopt_id, token_pos, locs, entry_kind);
+ EmitInstanceCallAOT(ic_data, deopt_id, token_pos, locs, entry_kind);
return;
}
ASSERT(!ic_data.IsNull());
@@ -1382,8 +1383,8 @@
return;
}
- EmitInstanceCall(StubEntryFor(ic_data, /*optimized=*/false), ic_data,
- deopt_id, token_pos, locs);
+ EmitInstanceCallJIT(StubEntryFor(ic_data, /*optimized=*/false), ic_data,
+ deopt_id, token_pos, locs, entry_kind);
}
void FlowGraphCompiler::GenerateStaticCall(intptr_t deopt_id,
@@ -2087,7 +2088,7 @@
zone(), original_call.ic_data()->AsUnaryClassChecks());
// TODO(sjindel/entrypoints): Support skiping type checks on switchable
// calls.
- EmitSwitchableInstanceCall(unary_checks, deopt_id, token_pos, locs);
+ EmitInstanceCallAOT(unary_checks, deopt_id, token_pos, locs);
}
}
}
diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.h b/runtime/vm/compiler/backend/flow_graph_compiler.h
index 28c369b..ca171b3 100644
--- a/runtime/vm/compiler/backend/flow_graph_compiler.h
+++ b/runtime/vm/compiler/backend/flow_graph_compiler.h
@@ -616,11 +616,12 @@
LocationSummary* locs,
Code::EntryKind entry_kind = Code::EntryKind::kNormal);
- void EmitInstanceCall(const Code& stub,
- const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs);
+ void EmitInstanceCallJIT(const Code& stub,
+ const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind);
void EmitPolymorphicInstanceCall(
const CallTargets& targets,
@@ -641,7 +642,7 @@
intptr_t try_index,
intptr_t slow_path_argument_count = 0);
- void EmitSwitchableInstanceCall(
+ void EmitInstanceCallAOT(
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc b/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc
index 9018e54..7b8e5e8 100644
--- a/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc
+++ b/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc
@@ -1066,17 +1066,26 @@
__ Drop(ic_data.CountWithTypeArgs());
}
-void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
- const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs) {
+void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
+ const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
+ ASSERT(entry_kind == Code::EntryKind::kNormal ||
+ entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
__ LoadFromOffset(kWord, R0, SP,
(ic_data.CountWithoutTypeArgs() - 1) * kWordSize);
__ LoadUniqueObject(R9, ic_data);
- GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
- Code::EntryKind::kMonomorphic);
+ __ LoadUniqueObject(CODE_REG, stub);
+ const intptr_t entry_point_offset =
+ entry_kind == Code::EntryKind::kNormal
+ ? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
+ : Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
+ __ ldr(LR, FieldAddress(CODE_REG, entry_point_offset));
+ __ blx(LR);
+ EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs());
}
@@ -1129,17 +1138,17 @@
__ Drop(args_desc.CountWithTypeArgs());
}
-void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs,
- Code::EntryKind entry_kind) {
+void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(ic_data.NumArgsTested() == 1);
const Code& initial_stub = StubCode::ICCallThroughFunction();
- __ Comment("SwitchableCall");
+ __ Comment("InstanceCallAOT");
__ LoadFromOffset(
kWord, R0, SP,
(ic_data.CountWithoutTypeArgs() - 1) * compiler::target::kWordSize);
diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc b/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc
index 505a96a..d632c20 100644
--- a/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc
+++ b/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc
@@ -1032,16 +1032,32 @@
__ Drop(ic_data.CountWithTypeArgs());
}
-void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
- const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs) {
+void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
+ const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
+ ASSERT(entry_kind == Code::EntryKind::kNormal ||
+ entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
__ LoadFromOffset(R0, SP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize);
- __ LoadUniqueObject(R5, ic_data);
- GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
- Code::EntryKind::kMonomorphic);
+
+ ObjectPoolBuilder& op = __ object_pool_builder();
+ const intptr_t ic_data_index =
+ op.AddObject(ic_data, ObjectPool::Patchability::kPatchable);
+ const intptr_t stub_index =
+ op.AddObject(stub, ObjectPool::Patchability::kPatchable);
+ ASSERT((ic_data_index + 1) == stub_index);
+ __ LoadDoubleWordFromPoolOffset(R5, CODE_REG,
+ ObjectPool::element_offset(ic_data_index));
+ const intptr_t entry_point_offset =
+ entry_kind == Code::EntryKind::kNormal
+ ? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
+ : Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
+ __ ldr(LR, FieldAddress(CODE_REG, entry_point_offset));
+ __ blr(LR);
+ EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs());
}
@@ -1090,18 +1106,18 @@
__ Drop(args_desc.CountWithTypeArgs());
}
-void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs,
- Code::EntryKind entry_kind) {
+void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
// TODO(34162): Support multiple entry-points on ARM64.
ASSERT(ic_data.NumArgsTested() == 1);
const Code& initial_stub = StubCode::ICCallThroughFunction();
- auto& op = __ object_pool_builder();
+ ObjectPoolBuilder& op = __ object_pool_builder();
- __ Comment("SwitchableCall");
+ __ Comment("InstanceCallAOT");
__ LoadFromOffset(R0, SP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize);
const intptr_t ic_data_index =
diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc b/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc
index 4cea80a..6cd2894 100644
--- a/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc
+++ b/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc
@@ -922,17 +922,25 @@
__ Drop(ic_data.CountWithTypeArgs());
}
-void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
- const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs) {
+void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
+ const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
+ ASSERT(entry_kind == Code::EntryKind::kNormal ||
+ entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(ic_data.arguments_descriptor()).Length() > 0);
// Load receiver into EBX.
__ movl(EBX, Address(ESP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize));
- __ LoadObject(ECX, ic_data);
- GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
- Code::EntryKind::kMonomorphic);
+ __ LoadObject(ECX, ic_data, true);
+ __ LoadObject(CODE_REG, stub, true);
+ const intptr_t entry_point_offset =
+ entry_kind == Code::EntryKind::kNormal
+ ? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
+ : Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
+ __ call(FieldAddress(CODE_REG, entry_point_offset));
+ EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs());
}
@@ -972,11 +980,11 @@
__ Drop(args_desc.CountWithTypeArgs());
}
-void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs,
- Code::EntryKind entry_kind) {
+void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
// Only generated with precompilation.
UNREACHABLE();
}
diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc b/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc
index 5542b19..5358ee0 100644
--- a/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc
+++ b/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc
@@ -1051,17 +1051,25 @@
__ Drop(ic_data.CountWithTypeArgs(), RCX);
}
-void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
- const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs) {
+void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
+ const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
+ ASSERT(entry_kind == Code::EntryKind::kNormal ||
+ entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
// Load receiver into RDX.
__ movq(RDX, Address(RSP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize));
__ LoadUniqueObject(RBX, ic_data);
- GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
- Code::EntryKind::kMonomorphic);
+ __ LoadUniqueObject(CODE_REG, stub);
+ const intptr_t entry_point_offset =
+ entry_kind == Code::EntryKind::kNormal
+ ? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
+ : Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
+ __ call(FieldAddress(CODE_REG, entry_point_offset));
+ EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs(), RCX);
}
@@ -1107,24 +1115,24 @@
__ Drop(args_desc.CountWithTypeArgs(), RCX);
}
-void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
- intptr_t deopt_id,
- TokenPosition token_pos,
- LocationSummary* locs,
- Code::EntryKind entry_kind) {
+void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
+ intptr_t deopt_id,
+ TokenPosition token_pos,
+ LocationSummary* locs,
+ Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(ic_data.NumArgsTested() == 1);
const Code& initial_stub = StubCode::ICCallThroughFunction();
- __ Comment("SwitchableCall");
+ __ Comment("InstanceCallAOT");
__ movq(RDX, Address(RSP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize));
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
// The AOT runtime will replace the slot in the object pool with the
// entrypoint address - see clustered_snapshot.cc.
__ LoadUniqueObject(RCX, initial_stub);
} else {
- intptr_t entry_point_offset =
+ const intptr_t entry_point_offset =
entry_kind == Code::EntryKind::kNormal
? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
: Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc
index 6c06e20..932b94e 100644
--- a/runtime/vm/compiler/backend/il.cc
+++ b/runtime/vm/compiler/backend/il.cc
@@ -723,6 +723,23 @@
return cids;
}
+static intptr_t Usage(const Function& function) {
+ intptr_t count = function.usage_counter();
+ if (count < 0) {
+ if (function.HasCode()) {
+ // 'function' is queued for optimized compilation
+ count = FLAG_optimization_counter_threshold;
+ } else {
+ // 'function' is queued for unoptimized compilation
+ count = FLAG_compilation_counter_threshold;
+ }
+ } else if (Code::IsOptimized(function.CurrentCode())) {
+ // 'function' was optimized and stopped counting
+ count = FLAG_optimization_counter_threshold;
+ }
+ return count;
+}
+
void Cids::CreateHelper(Zone* zone,
const ICData& ic_data,
int argument_number,
@@ -748,12 +765,36 @@
}
if (include_targets) {
Function& function = Function::ZoneHandle(zone, ic_data.GetTargetAt(i));
- cid_ranges_.Add(new (zone) TargetInfo(
- id, id, &function, ic_data.GetCountAt(i), ic_data.GetExactnessAt(i)));
+ intptr_t count = ic_data.GetCountAt(i);
+ cid_ranges_.Add(new (zone) TargetInfo(id, id, &function, count,
+ ic_data.GetExactnessAt(i)));
} else {
cid_ranges_.Add(new (zone) CidRange(id, id));
}
}
+
+ if (ic_data.is_megamorphic()) {
+ const MegamorphicCache& cache =
+ MegamorphicCache::Handle(zone, ic_data.AsMegamorphicCache());
+ SafepointMutexLocker ml(Isolate::Current()->megamorphic_lookup_mutex());
+ MegamorphicCacheEntries entries(Array::Handle(zone, cache.buckets()));
+ for (intptr_t i = 0; i < entries.Length(); i++) {
+ const intptr_t id =
+ Smi::Value(entries[i].Get<MegamorphicCache::kClassIdIndex>());
+ if (id == kIllegalCid) {
+ continue;
+ }
+ if (include_targets) {
+ Function& function = Function::ZoneHandle(zone);
+ function ^= entries[i].Get<MegamorphicCache::kTargetFunctionIndex>();
+ cid_ranges_.Add(new (zone) TargetInfo(
+ id, id, &function, Usage(function) / cache.filled_entry_count(),
+ StaticTypeExactnessState::NotTracking()));
+ } else {
+ cid_ranges_.Add(new (zone) CidRange(id, id));
+ }
+ }
+ }
}
bool Cids::IsMonomorphic() const {
@@ -3824,6 +3865,14 @@
Sort(OrderByFrequency);
}
+void CallTargets::Print() const {
+ for (intptr_t i = 0; i < length(); i++) {
+ THR_Print("cid = [%" Pd ", %" Pd "], count = %" Pd ", target = %s\n",
+ TargetAt(i)->cid_start, TargetAt(i)->cid_end, TargetAt(i)->count,
+ TargetAt(i)->target->ToQualifiedCString());
+ }
+}
+
// Shared code generation methods (EmitNativeCode and
// MakeLocationSummary). Only assembly code that can be shared across all
// architectures can be used. Machine specific register allocation and code
@@ -3914,7 +3963,11 @@
const Function& function = compiler->parsed_function().function();
if (function.IsDynamicFunction()) {
compiler->SpecialStatsBegin(CombinedCodeStatistics::kTagCheckedEntry);
- __ MonomorphicCheckedEntry();
+ if (!FLAG_precompiled_mode) {
+ __ MonomorphicCheckedEntryJIT();
+ } else {
+ __ MonomorphicCheckedEntryAOT();
+ }
compiler->SpecialStatsEnd(CombinedCodeStatistics::kTagCheckedEntry);
}
@@ -4349,8 +4402,8 @@
}
if (is_smi_two_args_op) {
ASSERT(ArgumentCount() == 2);
- compiler->EmitInstanceCall(stub, *call_ic_data, deopt_id(), token_pos(),
- locs());
+ compiler->EmitInstanceCallJIT(stub, *call_ic_data, deopt_id(),
+ token_pos(), locs(), entry_kind());
} else {
compiler->GenerateInstanceCall(deopt_id(), token_pos(), locs(),
*call_ic_data);
@@ -4532,8 +4585,8 @@
ASSERT(new_target->HasSingleTarget());
const Function& target = new_target->FirstTarget();
- StaticCallInstr* specialized =
- StaticCallInstr::FromCall(flow_graph->zone(), this, target);
+ StaticCallInstr* specialized = StaticCallInstr::FromCall(
+ flow_graph->zone(), this, target, new_target->AggregateCallCount());
flow_graph->InsertBefore(this, specialized, env(), FlowGraph::kValue);
return specialized;
}
diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h
index 757c185..21d4bf9 100644
--- a/runtime/vm/compiler/backend/il.h
+++ b/runtime/vm/compiler/backend/il.h
@@ -622,6 +622,8 @@
const Function& FirstTarget() const;
const Function& MostPopularTarget() const;
+ void Print() const;
+
private:
void MergeIntoRanges();
};
@@ -3982,7 +3984,8 @@
template <class C>
static StaticCallInstr* FromCall(Zone* zone,
const C* call,
- const Function& target) {
+ const Function& target,
+ intptr_t call_count) {
PushArgumentsArray* args =
new (zone) PushArgumentsArray(call->ArgumentCount());
for (intptr_t i = 0; i < call->ArgumentCount(); i++) {
@@ -3991,7 +3994,7 @@
StaticCallInstr* new_call = new (zone)
StaticCallInstr(call->token_pos(), target, call->type_args_len(),
call->argument_names(), args, call->deopt_id(),
- call->CallCount(), ICData::kNoRebind);
+ call_count, ICData::kNoRebind);
if (call->result_type() != NULL) {
new_call->result_type_ = call->result_type();
}
diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc
index c687ec4..d92ef09 100644
--- a/runtime/vm/compiler/backend/il_arm.cc
+++ b/runtime/vm/compiler/backend/il_arm.cc
@@ -3262,7 +3262,7 @@
LocationSummary* CheckStackOverflowInstr::MakeLocationSummary(Zone* zone,
bool opt) const {
const intptr_t kNumInputs = 0;
- const intptr_t kNumTemps = 1;
+ const intptr_t kNumTemps = 2;
const bool using_shared_stub = UseSharedSlowPathStub(opt);
ASSERT((kReservedCpuRegisters & (1 << LR)) != 0);
LocationSummary* summary = new (zone)
@@ -3270,6 +3270,7 @@
using_shared_stub ? LocationSummary::kCallOnSharedSlowPath
: LocationSummary::kCallOnSlowPath);
summary->set_temp(0, Location::RequiresRegister());
+ summary->set_temp(1, Location::RequiresRegister());
return summary;
}
@@ -3377,16 +3378,22 @@
compiler->AddSlowPathCode(slow_path);
__ b(slow_path->entry_label(), LS);
if (compiler->CanOSRFunction() && in_loop()) {
- const Register temp = locs()->temp(0).reg();
+ const Register function = locs()->temp(0).reg();
+ const Register count = locs()->temp(1).reg();
// In unoptimized code check the usage counter to trigger OSR at loop
// stack checks. Use progressively higher thresholds for more deeply
// nested loops to attempt to hit outer loops with OSR when possible.
- __ LoadObject(temp, compiler->parsed_function().function());
+ __ LoadObject(function, compiler->parsed_function().function());
intptr_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
- __ ldr(temp, FieldAddress(
- temp, compiler::target::Function::usage_counter_offset()));
- __ CompareImmediate(temp, threshold);
+ __ ldr(count,
+ FieldAddress(function,
+ compiler::target::Function::usage_counter_offset()));
+ __ add(count, count, Operand(1));
+ __ str(count,
+ FieldAddress(function,
+ compiler::target::Function::usage_counter_offset()));
+ __ CompareImmediate(count, threshold);
__ b(slow_path->osr_entry_label(), GE);
}
if (compiler->ForceSlowPathForStackOverflow()) {
diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc
index 1b5b2c3..154be8b 100644
--- a/runtime/vm/compiler/backend/il_arm64.cc
+++ b/runtime/vm/compiler/backend/il_arm64.cc
@@ -3004,15 +3004,19 @@
__ CompareRegisters(SP, TMP);
__ b(slow_path->entry_label(), LS);
if (compiler->CanOSRFunction() && in_loop()) {
- const Register temp = locs()->temp(0).reg();
+ const Register function = locs()->temp(0).reg();
// In unoptimized code check the usage counter to trigger OSR at loop
// stack checks. Use progressively higher thresholds for more deeply
// nested loops to attempt to hit outer loops with OSR when possible.
- __ LoadObject(temp, compiler->parsed_function().function());
+ __ LoadObject(function, compiler->parsed_function().function());
intptr_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
- __ LoadFieldFromOffset(temp, temp, Function::usage_counter_offset(), kWord);
- __ CompareImmediate(temp, threshold);
+ __ LoadFieldFromOffset(TMP, function, Function::usage_counter_offset(),
+ kWord);
+ __ add(TMP, TMP, Operand(1));
+ __ StoreFieldToOffset(TMP, function, Function::usage_counter_offset(),
+ kWord);
+ __ CompareImmediate(TMP, threshold);
__ b(slow_path->osr_entry_label(), GE);
}
if (compiler->ForceSlowPathForStackOverflow()) {
diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc
index 7268abe..77aee55 100644
--- a/runtime/vm/compiler/backend/il_ia32.cc
+++ b/runtime/vm/compiler/backend/il_ia32.cc
@@ -2858,6 +2858,7 @@
__ LoadObject(EDI, compiler->parsed_function().function());
intptr_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
+ __ incl(FieldAddress(EDI, Function::usage_counter_offset()));
__ cmpl(FieldAddress(EDI, Function::usage_counter_offset()),
Immediate(threshold));
__ j(GREATER_EQUAL, slow_path->osr_entry_label());
diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc
index c0d1f41..a3570c2 100644
--- a/runtime/vm/compiler/backend/il_x64.cc
+++ b/runtime/vm/compiler/backend/il_x64.cc
@@ -2989,6 +2989,7 @@
__ LoadObject(temp, compiler->parsed_function().function());
int32_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
+ __ incl(FieldAddress(temp, Function::usage_counter_offset()));
__ cmpl(FieldAddress(temp, Function::usage_counter_offset()),
Immediate(threshold));
__ j(GREATER_EQUAL, slow_path->osr_entry_label());
diff --git a/runtime/vm/compiler/call_specializer.cc b/runtime/vm/compiler/call_specializer.cc
index de3c1b5..7cf2dc1 100644
--- a/runtime/vm/compiler/call_specializer.cc
+++ b/runtime/vm/compiler/call_specializer.cc
@@ -314,7 +314,8 @@
ASSERT(targets->HasSingleTarget());
const Function& target = targets->FirstTarget();
- StaticCallInstr* specialized = StaticCallInstr::FromCall(Z, call, target);
+ StaticCallInstr* specialized =
+ StaticCallInstr::FromCall(Z, call, target, targets->AggregateCallCount());
call->ReplaceWith(specialized, current_iterator());
}
diff --git a/runtime/vm/compiler/jit/jit_call_specializer.cc b/runtime/vm/compiler/jit/jit_call_specializer.cc
index f073878..189ef8b 100644
--- a/runtime/vm/compiler/jit/jit_call_specializer.cc
+++ b/runtime/vm/compiler/jit/jit_call_specializer.cc
@@ -52,8 +52,10 @@
void JitCallSpecializer::ReplaceWithStaticCall(InstanceCallInstr* instr,
const ICData& unary_checks,
- const Function& target) {
- StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
+ const Function& target,
+ intptr_t call_count) {
+ StaticCallInstr* call =
+ StaticCallInstr::FromCall(Z, instr, target, call_count);
if (unary_checks.NumberOfChecks() == 1 &&
unary_checks.GetExactnessAt(0).IsExact()) {
if (unary_checks.GetExactnessAt(0).IsTriviallyExact()) {
@@ -143,7 +145,8 @@
Function::ZoneHandle(Z, unary_checks.GetTargetAt(0));
if (flow_graph()->CheckForInstanceCall(instr, target.kind()) ==
FlowGraph::ToCheck::kNoCheck) {
- ReplaceWithStaticCall(instr, unary_checks, target);
+ ReplaceWithStaticCall(instr, unary_checks, target,
+ targets.AggregateCallCount());
return;
}
}
@@ -168,7 +171,8 @@
// Call can still deoptimize, do not detach environment from instr.
const Function& target =
Function::ZoneHandle(Z, unary_checks.GetTargetAt(0));
- ReplaceWithStaticCall(instr, unary_checks, target);
+ ReplaceWithStaticCall(instr, unary_checks, target,
+ targets.AggregateCallCount());
} else {
PolymorphicInstanceCallInstr* call =
new (Z) PolymorphicInstanceCallInstr(instr, targets,
diff --git a/runtime/vm/compiler/jit/jit_call_specializer.h b/runtime/vm/compiler/jit/jit_call_specializer.h
index 77c72bd..00f7f67 100644
--- a/runtime/vm/compiler/jit/jit_call_specializer.h
+++ b/runtime/vm/compiler/jit/jit_call_specializer.h
@@ -37,7 +37,8 @@
void ReplaceWithStaticCall(InstanceCallInstr* instr,
const ICData& unary_checks,
- const Function& target);
+ const Function& target,
+ intptr_t call_count);
DISALLOW_COPY_AND_ASSIGN(JitCallSpecializer);
};
diff --git a/runtime/vm/compiler/runtime_api.h b/runtime/vm/compiler/runtime_api.h
index 57f4312..f6231ce 100644
--- a/runtime/vm/compiler/runtime_api.h
+++ b/runtime/vm/compiler/runtime_api.h
@@ -676,7 +676,6 @@
static word fix_allocation_stub_code_offset();
static word monomorphic_miss_stub_offset();
- static word ic_lookup_through_code_stub_offset();
static word lazy_specialize_type_test_stub_offset();
static word slow_type_test_stub_offset();
static word call_to_runtime_stub_offset();
@@ -775,8 +774,10 @@
class Instructions : public AllStatic {
public:
- static const word kPolymorphicEntryOffset;
- static const word kMonomorphicEntryOffset;
+ static const word kMonomorphicEntryOffsetJIT;
+ static const word kPolymorphicEntryOffsetJIT;
+ static const word kMonomorphicEntryOffsetAOT;
+ static const word kPolymorphicEntryOffsetAOT;
static word HeaderSize();
static word UnalignedHeaderSize();
};
diff --git a/runtime/vm/compiler/runtime_offsets_extracted.h b/runtime/vm/compiler/runtime_offsets_extracted.h
index 761b30f..4b1687a 100644
--- a/runtime/vm/compiler/runtime_offsets_extracted.h
+++ b/runtime/vm/compiler/runtime_offsets_extracted.h
@@ -25,9 +25,13 @@
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
3;
static constexpr dart::compiler::target::word
- Instructions_kMonomorphicEntryOffset = 20;
+ Instructions_kMonomorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
- Instructions_kPolymorphicEntryOffset = 0;
+ Instructions_kPolymorphicEntryOffsetJIT = 40;
+static constexpr dart::compiler::target::word
+ Instructions_kMonomorphicEntryOffsetAOT = 0;
+static constexpr dart::compiler::target::word
+ Instructions_kPolymorphicEntryOffsetAOT = 20;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@@ -187,7 +191,7 @@
static constexpr dart::compiler::target::word String_length_offset = 4;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4;
static constexpr dart::compiler::target::word
- Thread_AllocateArray_entry_point_offset = 284;
+ Thread_AllocateArray_entry_point_offset = 280;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
612;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@@ -195,97 +199,95 @@
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 112;
static constexpr dart::compiler::target::word
- Thread_array_write_barrier_entry_point_offset = 196;
+ Thread_array_write_barrier_entry_point_offset = 192;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
84;
static constexpr dart::compiler::target::word
- Thread_auto_scope_native_wrapper_entry_point_offset = 244;
+ Thread_auto_scope_native_wrapper_entry_point_offset = 240;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 104;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 100;
static constexpr dart::compiler::target::word
- Thread_call_to_runtime_entry_point_offset = 200;
+ Thread_call_to_runtime_entry_point_offset = 196;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 132;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 644;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
- 232;
+ 228;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
- 164;
+ 160;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
- 264;
+ 260;
static constexpr dart::compiler::target::word
- Thread_double_negate_address_offset = 260;
+ Thread_double_negate_address_offset = 256;
static constexpr dart::compiler::target::word Thread_end_offset = 60;
static constexpr dart::compiler::target::word
- Thread_enter_safepoint_stub_offset = 184;
+ Thread_enter_safepoint_stub_offset = 180;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
628;
static constexpr dart::compiler::target::word
- Thread_exit_safepoint_stub_offset = 188;
+ Thread_exit_safepoint_stub_offset = 184;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 120;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 116;
static constexpr dart::compiler::target::word
- Thread_float_absolute_address_offset = 276;
+ Thread_float_absolute_address_offset = 272;
static constexpr dart::compiler::target::word
- Thread_float_negate_address_offset = 272;
+ Thread_float_negate_address_offset = 268;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
- 268;
+ 264;
static constexpr dart::compiler::target::word
- Thread_float_zerow_address_offset = 280;
+ Thread_float_zerow_address_offset = 276;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
620;
static constexpr dart::compiler::target::word
- Thread_ic_lookup_through_code_stub_offset = 156;
-static constexpr dart::compiler::target::word
- Thread_interpret_call_entry_point_offset = 248;
+ Thread_interpret_call_entry_point_offset = 244;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 128;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 124;
static constexpr dart::compiler::target::word Thread_isolate_offset = 48;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_return_stub_offset = 168;
+ Thread_lazy_deopt_from_return_stub_offset = 164;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_throw_stub_offset = 172;
+ Thread_lazy_deopt_from_throw_stub_offset = 168;
static constexpr dart::compiler::target::word
- Thread_lazy_specialize_type_test_stub_offset = 180;
+ Thread_lazy_specialize_type_test_stub_offset = 176;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 72;
static constexpr dart::compiler::target::word
- Thread_megamorphic_call_checked_entry_offset = 220;
+ Thread_megamorphic_call_checked_entry_offset = 216;
static constexpr dart::compiler::target::word
- Thread_monomorphic_miss_entry_offset = 224;
+ Thread_monomorphic_miss_entry_offset = 220;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 152;
static constexpr dart::compiler::target::word
- Thread_no_scope_native_wrapper_entry_point_offset = 240;
+ Thread_no_scope_native_wrapper_entry_point_offset = 236;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_with_fpu_regs_entry_point_offset = 208;
+ Thread_null_error_shared_with_fpu_regs_entry_point_offset = 204;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 140;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_without_fpu_regs_entry_point_offset = 204;
+ Thread_null_error_shared_without_fpu_regs_entry_point_offset = 200;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 136;
static constexpr dart::compiler::target::word Thread_object_null_offset = 96;
static constexpr dart::compiler::target::word
- Thread_predefined_symbols_address_offset = 252;
+ Thread_predefined_symbols_address_offset = 248;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 624;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
632;
static constexpr dart::compiler::target::word
- Thread_slow_type_test_stub_offset = 176;
+ Thread_slow_type_test_stub_offset = 172;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 40;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 216;
+ Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 212;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 148;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 212;
+ Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 208;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 144;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@@ -300,11 +302,11 @@
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
108;
static constexpr dart::compiler::target::word
- Thread_write_barrier_entry_point_offset = 192;
+ Thread_write_barrier_entry_point_offset = 188;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
44;
static constexpr dart::compiler::target::word
- Thread_verify_callback_entry_offset = 236;
+ Thread_verify_callback_entry_offset = 232;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 636;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 12;
@@ -372,9 +374,13 @@
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
4;
static constexpr dart::compiler::target::word
- Instructions_kMonomorphicEntryOffset = 32;
+ Instructions_kMonomorphicEntryOffsetJIT = 8;
static constexpr dart::compiler::target::word
- Instructions_kPolymorphicEntryOffset = 8;
+ Instructions_kPolymorphicEntryOffsetJIT = 40;
+static constexpr dart::compiler::target::word
+ Instructions_kMonomorphicEntryOffsetAOT = 8;
+static constexpr dart::compiler::target::word
+ Instructions_kPolymorphicEntryOffsetAOT = 32;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@@ -535,7 +541,7 @@
static constexpr dart::compiler::target::word String_length_offset = 8;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8;
static constexpr dart::compiler::target::word
- Thread_AllocateArray_entry_point_offset = 560;
+ Thread_AllocateArray_entry_point_offset = 552;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
1232;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@@ -543,97 +549,95 @@
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 216;
static constexpr dart::compiler::target::word
- Thread_array_write_barrier_entry_point_offset = 384;
+ Thread_array_write_barrier_entry_point_offset = 376;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
168;
static constexpr dart::compiler::target::word
- Thread_auto_scope_native_wrapper_entry_point_offset = 480;
+ Thread_auto_scope_native_wrapper_entry_point_offset = 472;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 200;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 192;
static constexpr dart::compiler::target::word
- Thread_call_to_runtime_entry_point_offset = 392;
+ Thread_call_to_runtime_entry_point_offset = 384;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 256;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1296;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
- 456;
+ 448;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
- 320;
+ 312;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
- 520;
+ 512;
static constexpr dart::compiler::target::word
- Thread_double_negate_address_offset = 512;
+ Thread_double_negate_address_offset = 504;
static constexpr dart::compiler::target::word Thread_end_offset = 120;
static constexpr dart::compiler::target::word
- Thread_enter_safepoint_stub_offset = 360;
+ Thread_enter_safepoint_stub_offset = 352;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
1264;
static constexpr dart::compiler::target::word
- Thread_exit_safepoint_stub_offset = 368;
+ Thread_exit_safepoint_stub_offset = 360;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 232;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 224;
static constexpr dart::compiler::target::word
- Thread_float_absolute_address_offset = 544;
+ Thread_float_absolute_address_offset = 536;
static constexpr dart::compiler::target::word
- Thread_float_negate_address_offset = 536;
+ Thread_float_negate_address_offset = 528;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
- 528;
+ 520;
static constexpr dart::compiler::target::word
- Thread_float_zerow_address_offset = 552;
+ Thread_float_zerow_address_offset = 544;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
1248;
static constexpr dart::compiler::target::word
- Thread_ic_lookup_through_code_stub_offset = 304;
-static constexpr dart::compiler::target::word
- Thread_interpret_call_entry_point_offset = 488;
+ Thread_interpret_call_entry_point_offset = 480;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 248;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 240;
static constexpr dart::compiler::target::word Thread_isolate_offset = 96;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_return_stub_offset = 328;
+ Thread_lazy_deopt_from_return_stub_offset = 320;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_throw_stub_offset = 336;
+ Thread_lazy_deopt_from_throw_stub_offset = 328;
static constexpr dart::compiler::target::word
- Thread_lazy_specialize_type_test_stub_offset = 352;
+ Thread_lazy_specialize_type_test_stub_offset = 344;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 144;
static constexpr dart::compiler::target::word
- Thread_megamorphic_call_checked_entry_offset = 432;
+ Thread_megamorphic_call_checked_entry_offset = 424;
static constexpr dart::compiler::target::word
- Thread_monomorphic_miss_entry_offset = 440;
+ Thread_monomorphic_miss_entry_offset = 432;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 296;
static constexpr dart::compiler::target::word
- Thread_no_scope_native_wrapper_entry_point_offset = 472;
+ Thread_no_scope_native_wrapper_entry_point_offset = 464;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_with_fpu_regs_entry_point_offset = 408;
+ Thread_null_error_shared_with_fpu_regs_entry_point_offset = 400;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 272;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_without_fpu_regs_entry_point_offset = 400;
+ Thread_null_error_shared_without_fpu_regs_entry_point_offset = 392;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 264;
static constexpr dart::compiler::target::word Thread_object_null_offset = 184;
static constexpr dart::compiler::target::word
- Thread_predefined_symbols_address_offset = 496;
+ Thread_predefined_symbols_address_offset = 488;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1256;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
1272;
static constexpr dart::compiler::target::word
- Thread_slow_type_test_stub_offset = 344;
+ Thread_slow_type_test_stub_offset = 336;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 80;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 424;
+ Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 416;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 288;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 416;
+ Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 408;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 280;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@@ -648,11 +652,11 @@
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
208;
static constexpr dart::compiler::target::word
- Thread_write_barrier_entry_point_offset = 376;
+ Thread_write_barrier_entry_point_offset = 368;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
88;
static constexpr dart::compiler::target::word
- Thread_verify_callback_entry_offset = 464;
+ Thread_verify_callback_entry_offset = 456;
static constexpr dart::compiler::target::word Thread_callback_code_offset =
1280;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset =
@@ -722,9 +726,13 @@
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
3;
static constexpr dart::compiler::target::word
- Instructions_kMonomorphicEntryOffset = 0;
+ Instructions_kMonomorphicEntryOffsetJIT = 6;
static constexpr dart::compiler::target::word
- Instructions_kPolymorphicEntryOffset = 0;
+ Instructions_kPolymorphicEntryOffsetJIT = 34;
+static constexpr dart::compiler::target::word
+ Instructions_kMonomorphicEntryOffsetAOT = 0;
+static constexpr dart::compiler::target::word
+ Instructions_kPolymorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@@ -884,7 +892,7 @@
static constexpr dart::compiler::target::word String_length_offset = 4;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4;
static constexpr dart::compiler::target::word
- Thread_AllocateArray_entry_point_offset = 284;
+ Thread_AllocateArray_entry_point_offset = 280;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
576;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@@ -892,97 +900,95 @@
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 112;
static constexpr dart::compiler::target::word
- Thread_array_write_barrier_entry_point_offset = 196;
+ Thread_array_write_barrier_entry_point_offset = 192;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
84;
static constexpr dart::compiler::target::word
- Thread_auto_scope_native_wrapper_entry_point_offset = 244;
+ Thread_auto_scope_native_wrapper_entry_point_offset = 240;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 104;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 100;
static constexpr dart::compiler::target::word
- Thread_call_to_runtime_entry_point_offset = 200;
+ Thread_call_to_runtime_entry_point_offset = 196;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 132;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 608;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
- 232;
+ 228;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
- 164;
+ 160;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
- 264;
+ 260;
static constexpr dart::compiler::target::word
- Thread_double_negate_address_offset = 260;
+ Thread_double_negate_address_offset = 256;
static constexpr dart::compiler::target::word Thread_end_offset = 60;
static constexpr dart::compiler::target::word
- Thread_enter_safepoint_stub_offset = 184;
+ Thread_enter_safepoint_stub_offset = 180;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
592;
static constexpr dart::compiler::target::word
- Thread_exit_safepoint_stub_offset = 188;
+ Thread_exit_safepoint_stub_offset = 184;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 120;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 116;
static constexpr dart::compiler::target::word
- Thread_float_absolute_address_offset = 276;
+ Thread_float_absolute_address_offset = 272;
static constexpr dart::compiler::target::word
- Thread_float_negate_address_offset = 272;
+ Thread_float_negate_address_offset = 268;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
- 268;
+ 264;
static constexpr dart::compiler::target::word
- Thread_float_zerow_address_offset = 280;
+ Thread_float_zerow_address_offset = 276;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
584;
static constexpr dart::compiler::target::word
- Thread_ic_lookup_through_code_stub_offset = 156;
-static constexpr dart::compiler::target::word
- Thread_interpret_call_entry_point_offset = 248;
+ Thread_interpret_call_entry_point_offset = 244;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 128;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 124;
static constexpr dart::compiler::target::word Thread_isolate_offset = 48;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_return_stub_offset = 168;
+ Thread_lazy_deopt_from_return_stub_offset = 164;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_throw_stub_offset = 172;
+ Thread_lazy_deopt_from_throw_stub_offset = 168;
static constexpr dart::compiler::target::word
- Thread_lazy_specialize_type_test_stub_offset = 180;
+ Thread_lazy_specialize_type_test_stub_offset = 176;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 72;
static constexpr dart::compiler::target::word
- Thread_megamorphic_call_checked_entry_offset = 220;
+ Thread_megamorphic_call_checked_entry_offset = 216;
static constexpr dart::compiler::target::word
- Thread_monomorphic_miss_entry_offset = 224;
+ Thread_monomorphic_miss_entry_offset = 220;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 152;
static constexpr dart::compiler::target::word
- Thread_no_scope_native_wrapper_entry_point_offset = 240;
+ Thread_no_scope_native_wrapper_entry_point_offset = 236;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_with_fpu_regs_entry_point_offset = 208;
+ Thread_null_error_shared_with_fpu_regs_entry_point_offset = 204;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 140;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_without_fpu_regs_entry_point_offset = 204;
+ Thread_null_error_shared_without_fpu_regs_entry_point_offset = 200;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 136;
static constexpr dart::compiler::target::word Thread_object_null_offset = 96;
static constexpr dart::compiler::target::word
- Thread_predefined_symbols_address_offset = 252;
+ Thread_predefined_symbols_address_offset = 248;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 588;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
596;
static constexpr dart::compiler::target::word
- Thread_slow_type_test_stub_offset = 176;
+ Thread_slow_type_test_stub_offset = 172;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 40;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 216;
+ Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 212;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 148;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 212;
+ Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 208;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 144;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@@ -997,11 +1003,11 @@
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
108;
static constexpr dart::compiler::target::word
- Thread_write_barrier_entry_point_offset = 192;
+ Thread_write_barrier_entry_point_offset = 188;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
44;
static constexpr dart::compiler::target::word
- Thread_verify_callback_entry_offset = 236;
+ Thread_verify_callback_entry_offset = 232;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 600;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 12;
@@ -1065,9 +1071,13 @@
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
4;
static constexpr dart::compiler::target::word
- Instructions_kMonomorphicEntryOffset = 28;
+ Instructions_kMonomorphicEntryOffsetJIT = 8;
static constexpr dart::compiler::target::word
- Instructions_kPolymorphicEntryOffset = 8;
+ Instructions_kPolymorphicEntryOffsetJIT = 48;
+static constexpr dart::compiler::target::word
+ Instructions_kMonomorphicEntryOffsetAOT = 8;
+static constexpr dart::compiler::target::word
+ Instructions_kPolymorphicEntryOffsetAOT = 28;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@@ -1228,7 +1238,7 @@
static constexpr dart::compiler::target::word String_length_offset = 8;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8;
static constexpr dart::compiler::target::word
- Thread_AllocateArray_entry_point_offset = 560;
+ Thread_AllocateArray_entry_point_offset = 552;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
1320;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@@ -1236,97 +1246,95 @@
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 216;
static constexpr dart::compiler::target::word
- Thread_array_write_barrier_entry_point_offset = 384;
+ Thread_array_write_barrier_entry_point_offset = 376;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
168;
static constexpr dart::compiler::target::word
- Thread_auto_scope_native_wrapper_entry_point_offset = 480;
+ Thread_auto_scope_native_wrapper_entry_point_offset = 472;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 200;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 192;
static constexpr dart::compiler::target::word
- Thread_call_to_runtime_entry_point_offset = 392;
+ Thread_call_to_runtime_entry_point_offset = 384;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 256;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1384;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
- 456;
+ 448;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
- 320;
+ 312;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
- 520;
+ 512;
static constexpr dart::compiler::target::word
- Thread_double_negate_address_offset = 512;
+ Thread_double_negate_address_offset = 504;
static constexpr dart::compiler::target::word Thread_end_offset = 120;
static constexpr dart::compiler::target::word
- Thread_enter_safepoint_stub_offset = 360;
+ Thread_enter_safepoint_stub_offset = 352;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
1352;
static constexpr dart::compiler::target::word
- Thread_exit_safepoint_stub_offset = 368;
+ Thread_exit_safepoint_stub_offset = 360;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 232;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 224;
static constexpr dart::compiler::target::word
- Thread_float_absolute_address_offset = 544;
+ Thread_float_absolute_address_offset = 536;
static constexpr dart::compiler::target::word
- Thread_float_negate_address_offset = 536;
+ Thread_float_negate_address_offset = 528;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
- 528;
+ 520;
static constexpr dart::compiler::target::word
- Thread_float_zerow_address_offset = 552;
+ Thread_float_zerow_address_offset = 544;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
1336;
static constexpr dart::compiler::target::word
- Thread_ic_lookup_through_code_stub_offset = 304;
-static constexpr dart::compiler::target::word
- Thread_interpret_call_entry_point_offset = 488;
+ Thread_interpret_call_entry_point_offset = 480;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 248;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 240;
static constexpr dart::compiler::target::word Thread_isolate_offset = 96;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_return_stub_offset = 328;
+ Thread_lazy_deopt_from_return_stub_offset = 320;
static constexpr dart::compiler::target::word
- Thread_lazy_deopt_from_throw_stub_offset = 336;
+ Thread_lazy_deopt_from_throw_stub_offset = 328;
static constexpr dart::compiler::target::word
- Thread_lazy_specialize_type_test_stub_offset = 352;
+ Thread_lazy_specialize_type_test_stub_offset = 344;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 144;
static constexpr dart::compiler::target::word
- Thread_megamorphic_call_checked_entry_offset = 432;
+ Thread_megamorphic_call_checked_entry_offset = 424;
static constexpr dart::compiler::target::word
- Thread_monomorphic_miss_entry_offset = 440;
+ Thread_monomorphic_miss_entry_offset = 432;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 296;
static constexpr dart::compiler::target::word
- Thread_no_scope_native_wrapper_entry_point_offset = 472;
+ Thread_no_scope_native_wrapper_entry_point_offset = 464;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_with_fpu_regs_entry_point_offset = 408;
+ Thread_null_error_shared_with_fpu_regs_entry_point_offset = 400;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 272;
static constexpr dart::compiler::target::word
- Thread_null_error_shared_without_fpu_regs_entry_point_offset = 400;
+ Thread_null_error_shared_without_fpu_regs_entry_point_offset = 392;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 264;
static constexpr dart::compiler::target::word Thread_object_null_offset = 184;
static constexpr dart::compiler::target::word
- Thread_predefined_symbols_address_offset = 496;
+ Thread_predefined_symbols_address_offset = 488;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1344;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
1360;
static constexpr dart::compiler::target::word
- Thread_slow_type_test_stub_offset = 344;
+ Thread_slow_type_test_stub_offset = 336;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 80;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 424;
+ Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 416;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 288;
static constexpr dart::compiler::target::word
- Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 416;
+ Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 408;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 280;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@@ -1341,11 +1349,11 @@
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
208;
static constexpr dart::compiler::target::word
- Thread_write_barrier_entry_point_offset = 376;
+ Thread_write_barrier_entry_point_offset = 368;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
88;
static constexpr dart::compiler::target::word
- Thread_verify_callback_entry_offset = 464;
+ Thread_verify_callback_entry_offset = 456;
static constexpr dart::compiler::target::word Thread_callback_code_offset =
1368;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset =
@@ -1417,9 +1425,13 @@
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
4;
static constexpr dart::compiler::target::word
- Instructions_kMonomorphicEntryOffset = 0;
+ Instructions_kMonomorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
- Instructions_kPolymorphicEntryOffset = 0;
+ Instructions_kPolymorphicEntryOffsetJIT = 0;
+static constexpr dart::compiler::target::word
+ Instructions_kMonomorphicEntryOffsetAOT = 0;
+static constexpr dart::compiler::target::word
+ Instructions_kPolymorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@@ -1582,23 +1594,23 @@
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 296;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
- 880;
-static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
888;
+static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
+ 896;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
168;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 216;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 200;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 192;
-static constexpr dart::compiler::target::word Thread_dart_stream_offset = 944;
+static constexpr dart::compiler::target::word Thread_dart_stream_offset = 952;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
256;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 248;
static constexpr dart::compiler::target::word Thread_end_offset = 120;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
- 912;
+ 920;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 280;
static constexpr dart::compiler::target::word
@@ -1608,7 +1620,7 @@
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 288;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
- 896;
+ 904;
static constexpr dart::compiler::target::word Thread_isolate_offset = 96;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 144;
@@ -1617,9 +1629,9 @@
static constexpr dart::compiler::target::word Thread_object_null_offset = 184;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 232;
-static constexpr dart::compiler::target::word Thread_resume_pc_offset = 904;
+static constexpr dart::compiler::target::word Thread_resume_pc_offset = 912;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
- 920;
+ 928;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 80;
@@ -1634,7 +1646,7 @@
static constexpr dart::compiler::target::word Thread_vm_tag_offset = 160;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
88;
-static constexpr dart::compiler::target::word Thread_callback_code_offset = 928;
+static constexpr dart::compiler::target::word Thread_callback_code_offset = 936;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset =
16;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 16;
@@ -1698,9 +1710,13 @@
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
3;
static constexpr dart::compiler::target::word
- Instructions_kMonomorphicEntryOffset = 0;
+ Instructions_kMonomorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
- Instructions_kPolymorphicEntryOffset = 0;
+ Instructions_kPolymorphicEntryOffsetJIT = 0;
+static constexpr dart::compiler::target::word
+ Instructions_kMonomorphicEntryOffsetAOT = 0;
+static constexpr dart::compiler::target::word
+ Instructions_kPolymorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@@ -1862,23 +1878,23 @@
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 152;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
- 444;
-static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
448;
+static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
+ 452;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
84;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 112;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 104;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 100;
-static constexpr dart::compiler::target::word Thread_dart_stream_offset = 476;
+static constexpr dart::compiler::target::word Thread_dart_stream_offset = 480;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
132;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 128;
static constexpr dart::compiler::target::word Thread_end_offset = 60;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
- 460;
+ 464;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 144;
static constexpr dart::compiler::target::word
@@ -1888,7 +1904,7 @@
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 148;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
- 452;
+ 456;
static constexpr dart::compiler::target::word Thread_isolate_offset = 48;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 72;
@@ -1897,9 +1913,9 @@
static constexpr dart::compiler::target::word Thread_object_null_offset = 96;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 120;
-static constexpr dart::compiler::target::word Thread_resume_pc_offset = 456;
+static constexpr dart::compiler::target::word Thread_resume_pc_offset = 460;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
- 464;
+ 468;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 40;
@@ -1914,7 +1930,7 @@
static constexpr dart::compiler::target::word Thread_vm_tag_offset = 80;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
44;
-static constexpr dart::compiler::target::word Thread_callback_code_offset = 468;
+static constexpr dart::compiler::target::word Thread_callback_code_offset = 472;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 12;
static constexpr dart::compiler::target::word Type_arguments_offset = 16;
diff --git a/runtime/vm/compiler/runtime_offsets_list.h b/runtime/vm/compiler/runtime_offsets_list.h
index ab431e0..3f5ef94 100644
--- a/runtime/vm/compiler/runtime_offsets_list.h
+++ b/runtime/vm/compiler/runtime_offsets_list.h
@@ -35,8 +35,10 @@
CONSTANT(Array, kMaxElements) \
CONSTANT(Array, kMaxNewSpaceElements) \
CONSTANT(ClassTable, kSizeOfClassPairLog2) \
- CONSTANT(Instructions, kMonomorphicEntryOffset) \
- CONSTANT(Instructions, kPolymorphicEntryOffset) \
+ CONSTANT(Instructions, kMonomorphicEntryOffsetJIT) \
+ CONSTANT(Instructions, kPolymorphicEntryOffsetJIT) \
+ CONSTANT(Instructions, kMonomorphicEntryOffsetAOT) \
+ CONSTANT(Instructions, kPolymorphicEntryOffsetAOT) \
CONSTANT(HeapPage, kBytesPerCardLog2) \
CONSTANT(NativeEntry, kNumCallWrapperArguments) \
CONSTANT(String, kMaxElements) \
@@ -170,7 +172,6 @@
FIELD(Thread, float_not_address_offset) \
FIELD(Thread, float_zerow_address_offset) \
FIELD(Thread, global_object_pool_offset) \
- NOT_IN_DBC(FIELD(Thread, ic_lookup_through_code_stub_offset)) \
NOT_IN_DBC(FIELD(Thread, interpret_call_entry_point_offset)) \
NOT_IN_DBC(FIELD(Thread, invoke_dart_code_from_bytecode_stub_offset)) \
NOT_IN_DBC(FIELD(Thread, invoke_dart_code_stub_offset)) \
diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc
index bdf0ec4..f2fcc87 100644
--- a/runtime/vm/compiler/stub_code_compiler_arm.cc
+++ b/runtime/vm/compiler/stub_code_compiler_arm.cc
@@ -574,6 +574,9 @@
// (invalid because its function was optimized or deoptimized).
// R4: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
+ Label monomorphic;
+ __ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
+
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
@@ -593,6 +596,29 @@
// Jump to the dart function.
__ mov(CODE_REG, Operand(R0));
__ Branch(FieldAddress(R0, target::Code::entry_point_offset()));
+
+ __ Bind(&monomorphic);
+ // Load code pointer to this stub from the thread:
+ // The one that is passed in, is not correct - it points to the code object
+ // that needs to be replaced.
+ __ ldr(CODE_REG,
+ Address(THR, target::Thread::fix_callers_target_code_offset()));
+ // Create a stub frame as we are pushing some objects on the stack before
+ // calling into the runtime.
+ __ EnterStubFrame();
+ __ LoadImmediate(R1, 0);
+ __ Push(R9); // Preserve cache (guarded CID as Smi).
+ __ Push(R0); // Preserve receiver.
+ __ Push(R1);
+ __ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
+ __ Pop(CODE_REG);
+ __ Pop(R0); // Restore receiver.
+ __ Pop(R9); // Restore cache (guarded CID as Smi).
+ // Remove the stub frame.
+ __ LeaveStubFrame();
+ // Jump to the dart function.
+ __ Branch(FieldAddress(
+ CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}
// Called from object allocate instruction when the allocation stub has been
@@ -2414,6 +2440,22 @@
#endif // defined(PRODUCT)
}
+void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
+ Assembler* assembler) {
+#if defined(PRODUCT)
+ __ Stop("No debugging in PRODUCT mode");
+#else
+ __ EnterStubFrame();
+ __ Push(R9); // Preserve IC data.
+ __ PushImmediate(0); // Space for result.
+ __ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
+ __ Pop(CODE_REG); // Original stub.
+ __ Pop(R9); // Restore IC data.
+ __ LeaveStubFrame();
+ __ Branch(FieldAddress(CODE_REG, target::Code::entry_point_offset()));
+#endif // defined(PRODUCT)
+}
+
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
@@ -3196,17 +3238,17 @@
__ LoadImmediate(IP, 0);
__ Push(IP); // Result slot
- __ Push(R0); // Arg0: Receiver
- __ Push(R9); // Arg1: UnlinkedCall
- __ CallRuntime(kUnlinkedCallRuntimeEntry, 2);
+ __ Push(IP); // Arg0: stub out
+ __ Push(R0); // Arg1: Receiver
+ __ Push(R9); // Arg2: UnlinkedCall
+ __ CallRuntime(kUnlinkedCallRuntimeEntry, 3);
__ Drop(2);
+ __ Pop(CODE_REG); // result = stub
__ Pop(R9); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
- __ ldr(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ Branch(FieldAddress(
CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}
@@ -3239,16 +3281,16 @@
__ LoadImmediate(IP, 0);
__ Push(IP); // Result slot
- __ Push(R0); // Arg0: Receiver
- __ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
+ __ Push(IP); // Arg0: stub out
+ __ Push(R0); // Arg1: Receiver
+ __ CallRuntime(kSingleTargetMissRuntimeEntry, 2);
__ Drop(1);
+ __ Pop(CODE_REG); // result = stub
__ Pop(R9); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
- __ ldr(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ Branch(FieldAddress(
CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}
@@ -3263,16 +3305,16 @@
__ LoadImmediate(IP, 0);
__ Push(IP); // Result slot
- __ Push(R0); // Arg0: Receiver
- __ CallRuntime(kMonomorphicMissRuntimeEntry, 1);
+ __ Push(IP); // Arg0: stub out
+ __ Push(R0); // Arg1: Receiver
+ __ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
__ Drop(1);
+ __ Pop(CODE_REG); // result = stub
__ Pop(R9); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
- __ ldr(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ Branch(FieldAddress(
CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}
diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc
index 31e9286..7bebb69 100644
--- a/runtime/vm/compiler/stub_code_compiler_arm64.cc
+++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc
@@ -618,6 +618,9 @@
// (invalid because its function was optimized or deoptimized).
// R4: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
+ Label monomorphic;
+ __ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
+
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
@@ -638,6 +641,30 @@
// Jump to the dart function.
__ LoadFieldFromOffset(R0, CODE_REG, target::Code::entry_point_offset());
__ br(R0);
+
+ __ Bind(&monomorphic);
+ // Load code pointer to this stub from the thread:
+ // The one that is passed in, is not correct - it points to the code object
+ // that needs to be replaced.
+ __ ldr(CODE_REG,
+ Address(THR, target::Thread::fix_callers_target_code_offset()));
+ // Create a stub frame as we are pushing some objects on the stack before
+ // calling into the runtime.
+ __ EnterStubFrame();
+ __ Push(R5); // Preserve cache (guarded CID as Smi).
+ __ Push(R0); // Preserve receiver.
+ __ Push(ZR);
+ __ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
+ __ Pop(CODE_REG);
+ __ Pop(R0); // Restore receiver.
+ __ Pop(R5); // Restore cache (guarded CID as Smi).
+ // Remove the stub frame.
+ __ LeaveStubFrame();
+ // Jump to the dart function.
+ __ LoadFieldFromOffset(
+ R1, CODE_REG,
+ target::Code::entry_point_offset(CodeEntryKind::kMonomorphic));
+ __ br(R1);
}
// Called from object allocate instruction when the allocation stub has been
@@ -2489,6 +2516,23 @@
#endif // defined(PRODUCT)
}
+void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
+ Assembler* assembler) {
+#if defined(PRODUCT)
+ __ Stop("No debugging in PRODUCT mode");
+#else
+ __ EnterStubFrame();
+ __ Push(R5); // Preserve IC data.
+ __ Push(ZR); // Space for result.
+ __ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
+ __ Pop(CODE_REG); // Original stub.
+ __ Pop(R5); // Restore IC data.
+ __ LeaveStubFrame();
+ __ LoadFieldFromOffset(TMP, CODE_REG, target::Code::entry_point_offset());
+ __ br(TMP);
+#endif // defined(PRODUCT)
+}
+
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
@@ -3265,17 +3309,17 @@
__ Push(R0); // Preserve receiver.
__ Push(ZR); // Result slot.
- __ Push(R0); // Arg0: Receiver
- __ Push(R5); // Arg1: UnlinkedCall
- __ CallRuntime(kUnlinkedCallRuntimeEntry, 2);
+ __ Push(ZR); // Arg0: stub out.
+ __ Push(R0); // Arg1: Receiver
+ __ Push(R5); // Arg2: UnlinkedCall
+ __ CallRuntime(kUnlinkedCallRuntimeEntry, 3);
__ Drop(2);
+ __ Pop(CODE_REG); // result = stub
__ Pop(R5); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
- __ ldr(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ br(R1);
@@ -3309,16 +3353,16 @@
__ Push(R0); // Preserve receiver.
__ Push(ZR); // Result slot.
- __ Push(R0); // Arg0: Receiver
- __ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
+ __ Push(ZR); // Arg0: Stub out.
+ __ Push(R0); // Arg1: Receiver
+ __ CallRuntime(kSingleTargetMissRuntimeEntry, 2);
__ Drop(1);
+ __ Pop(CODE_REG); // result = stub
__ Pop(R5); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
- __ ldr(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ br(R1);
@@ -3333,16 +3377,16 @@
__ Push(R0); // Preserve receiver.
__ Push(ZR); // Result slot.
- __ Push(R0); // Arg0: Receiver
- __ CallRuntime(kMonomorphicMissRuntimeEntry, 1);
+ __ Push(ZR); // Arg0: stub out
+ __ Push(R0); // Arg1: Receiver
+ __ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
__ Drop(1);
+ __ Pop(CODE_REG); // result = stub
__ Pop(R5); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
- __ ldr(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ br(R1);
diff --git a/runtime/vm/compiler/stub_code_compiler_ia32.cc b/runtime/vm/compiler/stub_code_compiler_ia32.cc
index 8e59855..dc4e3ec 100644
--- a/runtime/vm/compiler/stub_code_compiler_ia32.cc
+++ b/runtime/vm/compiler/stub_code_compiler_ia32.cc
@@ -407,8 +407,10 @@
// (invalid because its function was optimized or deoptimized).
// EDX: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
- // Create a stub frame as we are pushing some objects on the stack before
- // calling into the runtime.
+ Label monomorphic;
+ __ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
+
+ // This was a static call.
__ EnterStubFrame();
__ pushl(EDX); // Preserve arguments descriptor array.
__ pushl(Immediate(0)); // Setup space on stack for return value.
@@ -419,6 +421,22 @@
__ LeaveFrame();
__ jmp(EAX);
__ int3();
+
+ __ Bind(&monomorphic);
+ // This was a switchable call.
+ __ EnterStubFrame();
+ __ pushl(ECX); // Preserve cache (guarded CID as Smi).
+ __ pushl(EBX); // Preserve receiver.
+ __ pushl(Immediate(0)); // Result slot.
+ __ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
+ __ popl(CODE_REG); // Get Code object.
+ __ popl(EBX); // Restore receiver.
+ __ popl(ECX); // Restore cache (guarded CID as Smi).
+ __ movl(EAX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
+ CodeEntryKind::kMonomorphic)));
+ __ LeaveFrame();
+ __ jmp(EAX);
+ __ int3();
}
// Called from object allocate instruction when the allocation stub has been
@@ -2119,6 +2137,23 @@
#endif // defined(PRODUCT)
}
+void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
+ Assembler* assembler) {
+#if defined(PRODUCT)
+ __ Stop("No debugging in PRODUCT mode");
+#else
+ __ EnterStubFrame();
+ __ pushl(ECX); // Preserve ICData.
+ __ pushl(Immediate(0)); // Room for result.
+ __ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
+ __ popl(EAX); // Code of original stub.
+ __ popl(ECX); // Restore ICData.
+ __ LeaveFrame();
+ // Jump to original stub.
+ __ jmp(FieldAddress(EAX, target::Code::entry_point_offset()));
+#endif // defined(PRODUCT)
+}
+
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
@@ -2661,7 +2696,23 @@
}
void StubCodeCompiler::GenerateMonomorphicMissStub(Assembler* assembler) {
- __ int3(); // AOT only.
+ __ EnterStubFrame();
+ __ pushl(EBX); // Preserve receiver.
+
+ __ pushl(Immediate(0)); // Result slot.
+ __ pushl(Immediate(0)); // Arg0: stub out
+ __ pushl(EBX); // Arg1: Receiver
+ __ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
+ __ popl(ECX);
+ __ popl(CODE_REG); // result = stub
+ __ popl(ECX); // result = IC
+
+ __ popl(EBX); // Restore receiver.
+ __ LeaveFrame();
+
+ __ movl(EAX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
+ CodeEntryKind::kMonomorphic)));
+ __ jmp(EAX);
}
void StubCodeCompiler::GenerateFrameAwaitingMaterializationStub(
diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc
index 77e894f..0402147 100644
--- a/runtime/vm/compiler/stub_code_compiler_x64.cc
+++ b/runtime/vm/compiler/stub_code_compiler_x64.cc
@@ -571,6 +571,10 @@
// (invalid because its function was optimized or deoptimized).
// R10: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
+ Label monomorphic;
+ __ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
+
+ // This was a static call.
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
@@ -587,6 +591,27 @@
__ LeaveStubFrame();
__ jmp(RAX);
__ int3();
+
+ __ Bind(&monomorphic);
+ // This was a switchable call.
+ // Load code pointer to this stub from the thread:
+ // The one that is passed in, is not correct - it points to the code object
+ // that needs to be replaced.
+ __ movq(CODE_REG,
+ Address(THR, target::Thread::fix_callers_target_code_offset()));
+ __ EnterStubFrame();
+ __ pushq(RBX); // Preserve cache (guarded CID as Smi).
+ __ pushq(RDX); // Preserve receiver.
+ __ pushq(Immediate(0)); // Result slot.
+ __ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
+ __ popq(CODE_REG); // Get Code object.
+ __ popq(RDX); // Restore receiver.
+ __ popq(RBX); // Restore cache (guarded CID as Smi).
+ __ movq(RAX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
+ CodeEntryKind::kMonomorphic)));
+ __ LeaveStubFrame();
+ __ jmp(RAX);
+ __ int3();
}
// Called from object allocate instruction when the allocation stub has been
@@ -2500,6 +2525,26 @@
#endif // defined(PRODUCT)
}
+void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
+ Assembler* assembler) {
+#if defined(PRODUCT)
+ __ Stop("No debugging in PRODUCT mode");
+#else
+ __ EnterStubFrame();
+ __ pushq(RDX); // Preserve receiver.
+ __ pushq(RBX); // Preserve IC data.
+ __ pushq(Immediate(0)); // Result slot.
+ __ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
+ __ popq(CODE_REG); // Original stub.
+ __ popq(RBX); // Restore IC data.
+ __ popq(RDX); // Restore receiver.
+ __ LeaveStubFrame();
+
+ __ movq(RAX, FieldAddress(CODE_REG, target::Code::entry_point_offset()));
+ __ jmp(RAX); // Jump to original stub.
+#endif // defined(PRODUCT)
+}
+
// TOS(0): return address (Dart code).
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
@@ -3284,18 +3329,18 @@
__ pushq(RDX); // Preserve receiver.
__ pushq(Immediate(0)); // Result slot.
- __ pushq(RDX); // Arg0: Receiver
- __ pushq(RBX); // Arg1: UnlinkedCall
- __ CallRuntime(kUnlinkedCallRuntimeEntry, 2);
+ __ pushq(Immediate(0)); // Arg0: stub out.
+ __ pushq(RDX); // Arg1: Receiver
+ __ pushq(RBX); // Arg2: UnlinkedCall
+ __ CallRuntime(kUnlinkedCallRuntimeEntry, 3);
__ popq(RBX);
__ popq(RBX);
+ __ popq(CODE_REG); // result = stub
__ popq(RBX); // result = IC
__ popq(RDX); // Restore receiver.
__ LeaveStubFrame();
- __ movq(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ movq(RCX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ jmp(RCX);
@@ -3328,16 +3373,16 @@
__ pushq(RDX); // Preserve receiver.
__ pushq(Immediate(0)); // Result slot.
- __ pushq(RDX); // Arg0: Receiver
- __ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
+ __ pushq(Immediate(0)); // Arg0: stub out
+ __ pushq(RDX); // Arg1: Receiver
+ __ CallRuntime(kSingleTargetMissRuntimeEntry, 2);
__ popq(RBX);
+ __ popq(CODE_REG); // result = stub
__ popq(RBX); // result = IC
__ popq(RDX); // Restore receiver.
__ LeaveStubFrame();
- __ movq(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ movq(RCX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ jmp(RCX);
@@ -3352,16 +3397,16 @@
__ pushq(RDX); // Preserve receiver.
__ pushq(Immediate(0)); // Result slot.
- __ pushq(RDX); // Arg0: Receiver
- __ CallRuntime(kMonomorphicMissRuntimeEntry, 1);
+ __ pushq(Immediate(0)); // Arg0: stub out.
+ __ pushq(RDX); // Arg1: Receiver
+ __ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
__ popq(RBX);
+ __ popq(CODE_REG); // result = stub
__ popq(RBX); // result = IC
__ popq(RDX); // Restore receiver.
__ LeaveStubFrame();
- __ movq(CODE_REG,
- Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ movq(RCX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ jmp(RCX);
diff --git a/runtime/vm/debugger.cc b/runtime/vm/debugger.cc
index f79f581..10412c0 100644
--- a/runtime/vm/debugger.cc
+++ b/runtime/vm/debugger.cc
@@ -1860,18 +1860,26 @@
// that inline the function that contains the newly created breakpoint.
// We currently don't have this info so we deoptimize all functions.
void Debugger::DeoptimizeWorld() {
+#if defined(DART_PRECOMPILED_RUNTIME)
+ UNREACHABLE();
+#else
BackgroundCompiler::Stop(isolate_);
if (FLAG_trace_deoptimization) {
THR_Print("Deopt for debugger\n");
}
+ isolate_->set_has_attempted_stepping(true);
+
DeoptimizeFunctionsOnStack();
+
// Iterate over all classes, deoptimize functions.
// TODO(hausner): Could possibly be combined with RemoveOptimizedCode()
const ClassTable& class_table = *isolate_->class_table();
- Class& cls = Class::Handle();
- Array& functions = Array::Handle();
- GrowableObjectArray& closures = GrowableObjectArray::Handle();
- Function& function = Function::Handle();
+ Zone* zone = Thread::Current()->zone();
+ Class& cls = Class::Handle(zone);
+ Array& functions = Array::Handle(zone);
+ GrowableObjectArray& closures = GrowableObjectArray::Handle(zone);
+ Function& function = Function::Handle(zone);
+ Code& code = Code::Handle(zone);
intptr_t num_classes = class_table.NumCids();
for (intptr_t i = 1; i < num_classes; i++) {
if (class_table.HasValidClassAt(i)) {
@@ -1887,12 +1895,20 @@
if (function.HasOptimizedCode()) {
function.SwitchToUnoptimizedCode();
}
+ code = function.unoptimized_code();
+ if (!code.IsNull()) {
+ code.ResetSwitchableCalls(zone);
+ }
// Also disable any optimized implicit closure functions.
if (function.HasImplicitClosureFunction()) {
function = function.ImplicitClosureFunction();
if (function.HasOptimizedCode()) {
function.SwitchToUnoptimizedCode();
}
+ code = function.unoptimized_code();
+ if (!code.IsNull()) {
+ code.ResetSwitchableCalls(zone);
+ }
}
}
}
@@ -1908,7 +1924,12 @@
if (function.HasOptimizedCode()) {
function.SwitchToUnoptimizedCode();
}
+ code = function.unoptimized_code();
+ if (!code.IsNull()) {
+ code.ResetSwitchableCalls(zone);
+ }
}
+#endif // defined(DART_PRECOMPILED_RUNTIME)
}
void Debugger::NotifySingleStepping(bool value) const {
diff --git a/runtime/vm/debugger_arm.cc b/runtime/vm/debugger_arm.cc
index 3eafa74..15d95ba 100644
--- a/runtime/vm/debugger_arm.cc
+++ b/runtime/vm/debugger_arm.cc
@@ -24,9 +24,11 @@
Code& stub_target = Code::Handle();
switch (breakpoint_kind_) {
case RawPcDescriptors::kIcCall:
- case RawPcDescriptors::kUnoptStaticCall:
stub_target = StubCode::ICCallBreakpoint().raw();
break;
+ case RawPcDescriptors::kUnoptStaticCall:
+ stub_target = StubCode::UnoptStaticCallBreakpoint().raw();
+ break;
case RawPcDescriptors::kRuntimeCall:
stub_target = StubCode::RuntimeCallBreakpoint().raw();
break;
diff --git a/runtime/vm/debugger_arm64.cc b/runtime/vm/debugger_arm64.cc
index 48ed9c2..e9748ca 100644
--- a/runtime/vm/debugger_arm64.cc
+++ b/runtime/vm/debugger_arm64.cc
@@ -21,22 +21,30 @@
void CodeBreakpoint::PatchCode() {
ASSERT(!is_enabled_);
- Code& stub_target = Code::Handle();
+ const Code& code = Code::Handle(code_);
switch (breakpoint_kind_) {
- case RawPcDescriptors::kIcCall:
- case RawPcDescriptors::kUnoptStaticCall:
- stub_target = StubCode::ICCallBreakpoint().raw();
+ case RawPcDescriptors::kIcCall: {
+ Object& data = Object::Handle();
+ saved_value_ = CodePatcher::GetInstanceCallAt(pc_, code, &data);
+ CodePatcher::PatchInstanceCallAt(pc_, code, data,
+ StubCode::ICCallBreakpoint());
break;
+ }
+ case RawPcDescriptors::kUnoptStaticCall: {
+ saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
+ CodePatcher::PatchPoolPointerCallAt(
+ pc_, code, StubCode::UnoptStaticCallBreakpoint());
+ break;
+ }
case RawPcDescriptors::kRuntimeCall: {
- stub_target = StubCode::RuntimeCallBreakpoint().raw();
+ saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
+ CodePatcher::PatchPoolPointerCallAt(pc_, code,
+ StubCode::RuntimeCallBreakpoint());
break;
}
default:
UNREACHABLE();
}
- const Code& code = Code::Handle(code_);
- saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
- CodePatcher::PatchPoolPointerCallAt(pc_, code, stub_target);
is_enabled_ = true;
}
@@ -44,7 +52,13 @@
ASSERT(is_enabled_);
const Code& code = Code::Handle(code_);
switch (breakpoint_kind_) {
- case RawPcDescriptors::kIcCall:
+ case RawPcDescriptors::kIcCall: {
+ Object& data = Object::Handle();
+ CodePatcher::GetInstanceCallAt(pc_, code, &data);
+ CodePatcher::PatchInstanceCallAt(pc_, code, data,
+ Code::Handle(saved_value_));
+ break;
+ }
case RawPcDescriptors::kUnoptStaticCall:
case RawPcDescriptors::kRuntimeCall: {
CodePatcher::PatchPoolPointerCallAt(pc_, code,
diff --git a/runtime/vm/debugger_ia32.cc b/runtime/vm/debugger_ia32.cc
index d74202c..10ee78f 100644
--- a/runtime/vm/debugger_ia32.cc
+++ b/runtime/vm/debugger_ia32.cc
@@ -31,11 +31,14 @@
{
WritableInstructionsScope writable(instrs.PayloadStart(), instrs.Size());
switch (breakpoint_kind_) {
- case RawPcDescriptors::kIcCall:
- case RawPcDescriptors::kUnoptStaticCall: {
+ case RawPcDescriptors::kIcCall: {
stub_target = StubCode::ICCallBreakpoint().raw();
break;
}
+ case RawPcDescriptors::kUnoptStaticCall: {
+ stub_target = StubCode::UnoptStaticCallBreakpoint().raw();
+ break;
+ }
case RawPcDescriptors::kRuntimeCall: {
saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
stub_target = StubCode::RuntimeCallBreakpoint().raw();
diff --git a/runtime/vm/debugger_x64.cc b/runtime/vm/debugger_x64.cc
index 39cf91c..883880f 100644
--- a/runtime/vm/debugger_x64.cc
+++ b/runtime/vm/debugger_x64.cc
@@ -26,9 +26,11 @@
Code& stub_target = Code::Handle();
switch (breakpoint_kind_) {
case RawPcDescriptors::kIcCall:
- case RawPcDescriptors::kUnoptStaticCall:
stub_target = StubCode::ICCallBreakpoint().raw();
break;
+ case RawPcDescriptors::kUnoptStaticCall:
+ stub_target = StubCode::UnoptStaticCallBreakpoint().raw();
+ break;
case RawPcDescriptors::kRuntimeCall:
stub_target = StubCode::RuntimeCallBreakpoint().raw();
break;
diff --git a/runtime/vm/deferred_objects.cc b/runtime/vm/deferred_objects.cc
index 76ca7b7..7238899 100644
--- a/runtime/vm/deferred_objects.cc
+++ b/runtime/vm/deferred_objects.cc
@@ -124,15 +124,15 @@
if (pc != 0) {
// If the deoptimization happened at an IC call, update the IC data
// to avoid repeated deoptimization at the same site next time around.
- ICData& ic_data = ICData::Handle(zone);
- CodePatcher::GetInstanceCallAt(pc, code, &ic_data);
- if (!ic_data.IsNull()) {
- ic_data.AddDeoptReason(deopt_context->deopt_reason());
- // Propagate the reason to all ICData-s with same deopt_id since
- // only unoptimized-code ICData (IC calls) are propagated.
- function.SetDeoptReasonForAll(ic_data.deopt_id(),
- deopt_context->deopt_reason());
- }
+ // We cannot use CodePatcher::GetInstanceCallAt because the call site
+ // may have switched to from referencing an ICData to a target Code or
+ // MegamorphicCache.
+ ICData& ic_data = ICData::Handle(zone, function.FindICData(deopt_id_));
+ ic_data.AddDeoptReason(deopt_context->deopt_reason());
+ // Propagate the reason to all ICData-s with same deopt_id since
+ // only unoptimized-code ICData (IC calls) are propagated.
+ function.SetDeoptReasonForAll(ic_data.deopt_id(),
+ deopt_context->deopt_reason());
} else {
if (deopt_context->HasDeoptFlag(ICData::kHoisted)) {
// Prevent excessive deoptimization.
diff --git a/runtime/vm/instructions_arm.cc b/runtime/vm/instructions_arm.cc
index c8e8af9..c2be95c 100644
--- a/runtime/vm/instructions_arm.cc
+++ b/runtime/vm/instructions_arm.cc
@@ -18,20 +18,35 @@
CallPattern::CallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
- end_(pc),
- ic_data_load_end_(0),
- target_code_pool_index_(-1),
- ic_data_(ICData::Handle()) {
+ target_code_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(pc));
// Last instruction: blx lr.
- ASSERT(*(reinterpret_cast<uword*>(end_) - 1) == 0xe12fff3e);
+ ASSERT(*(reinterpret_cast<uword*>(pc) - 1) == 0xe12fff3e);
Register reg;
- ic_data_load_end_ = InstructionPattern::DecodeLoadWordFromPool(
- end_ - 2 * Instr::kInstrSize, ®, &target_code_pool_index_);
+ InstructionPattern::DecodeLoadWordFromPool(pc - 2 * Instr::kInstrSize, ®,
+ &target_code_pool_index_);
ASSERT(reg == CODE_REG);
}
+ICCallPattern::ICCallPattern(uword pc, const Code& code)
+ : object_pool_(ObjectPool::Handle(code.GetObjectPool())),
+ target_pool_index_(-1),
+ data_pool_index_(-1) {
+ ASSERT(code.ContainsInstructionAt(pc));
+ // Last instruction: blx lr.
+ ASSERT(*(reinterpret_cast<uword*>(pc) - 1) == 0xe12fff3e);
+
+ Register reg;
+ uword data_load_end = InstructionPattern::DecodeLoadWordFromPool(
+ pc - 2 * Instr::kInstrSize, ®, &target_pool_index_);
+ ASSERT(reg == CODE_REG);
+
+ InstructionPattern::DecodeLoadWordFromPool(data_load_end, ®,
+ &data_pool_index_);
+ ASSERT(reg == R9);
+}
+
NativeCallPattern::NativeCallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
@@ -245,16 +260,6 @@
return false;
}
-RawICData* CallPattern::IcData() {
- if (ic_data_.IsNull()) {
- Register reg;
- InstructionPattern::DecodeLoadObject(ic_data_load_end_, object_pool_, ®,
- &ic_data_);
- ASSERT(reg == R9);
- }
- return ic_data_.raw();
-}
-
RawCode* CallPattern::TargetCode() const {
return reinterpret_cast<RawCode*>(
object_pool_.ObjectAt(target_code_pool_index_));
@@ -264,6 +269,23 @@
object_pool_.SetObjectAt(target_code_pool_index_, target_code);
}
+RawObject* ICCallPattern::Data() const {
+ return object_pool_.ObjectAt(data_pool_index_);
+}
+
+void ICCallPattern::SetData(const Object& data) const {
+ ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
+ object_pool_.SetObjectAt(data_pool_index_, data);
+}
+
+RawCode* ICCallPattern::TargetCode() const {
+ return reinterpret_cast<RawCode*>(object_pool_.ObjectAt(target_pool_index_));
+}
+
+void ICCallPattern::SetTargetCode(const Code& target_code) const {
+ object_pool_.SetObjectAt(target_pool_index_, target_code);
+}
+
SwitchableCallPatternBase::SwitchableCallPatternBase(const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
data_pool_index_(-1),
diff --git a/runtime/vm/instructions_arm.h b/runtime/vm/instructions_arm.h
index d3e1cdb..bbd0d6c 100644
--- a/runtime/vm/instructions_arm.h
+++ b/runtime/vm/instructions_arm.h
@@ -72,7 +72,23 @@
public:
CallPattern(uword pc, const Code& code);
- RawICData* IcData();
+ RawCode* TargetCode() const;
+ void SetTargetCode(const Code& code) const;
+
+ private:
+ const ObjectPool& object_pool_;
+
+ intptr_t target_code_pool_index_;
+
+ DISALLOW_COPY_AND_ASSIGN(CallPattern);
+};
+
+class ICCallPattern : public ValueObject {
+ public:
+ ICCallPattern(uword pc, const Code& code);
+
+ RawObject* Data() const;
+ void SetData(const Object& data) const;
RawCode* TargetCode() const;
void SetTargetCode(const Code& code) const;
@@ -80,13 +96,10 @@
private:
const ObjectPool& object_pool_;
- uword end_;
- uword ic_data_load_end_;
+ intptr_t target_pool_index_;
+ intptr_t data_pool_index_;
- intptr_t target_code_pool_index_;
- ICData& ic_data_;
-
- DISALLOW_COPY_AND_ASSIGN(CallPattern);
+ DISALLOW_COPY_AND_ASSIGN(ICCallPattern);
};
class NativeCallPattern : public ValueObject {
diff --git a/runtime/vm/instructions_arm64.cc b/runtime/vm/instructions_arm64.cc
index cc29bcd..a421c27 100644
--- a/runtime/vm/instructions_arm64.cc
+++ b/runtime/vm/instructions_arm64.cc
@@ -18,20 +18,36 @@
CallPattern::CallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
- end_(pc),
- ic_data_load_end_(0),
- target_code_pool_index_(-1),
- ic_data_(ICData::Handle()) {
+ target_code_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(pc));
// Last instruction: blr ip0.
- ASSERT(*(reinterpret_cast<uint32_t*>(end_) - 1) == 0xd63f0200);
+ ASSERT(*(reinterpret_cast<uint32_t*>(pc) - 1) == 0xd63f0200);
Register reg;
- ic_data_load_end_ = InstructionPattern::DecodeLoadWordFromPool(
- end_ - 2 * Instr::kInstrSize, ®, &target_code_pool_index_);
+ InstructionPattern::DecodeLoadWordFromPool(pc - 2 * Instr::kInstrSize, ®,
+ &target_code_pool_index_);
ASSERT(reg == CODE_REG);
}
+ICCallPattern::ICCallPattern(uword pc, const Code& code)
+ : object_pool_(ObjectPool::Handle(code.GetObjectPool())),
+ target_pool_index_(-1),
+ data_pool_index_(-1) {
+ ASSERT(code.ContainsInstructionAt(pc));
+ // Last instruction: blr lr.
+ ASSERT(*(reinterpret_cast<uint32_t*>(pc) - 1) == 0xd63f03c0);
+
+ Register data_reg, code_reg;
+ intptr_t pool_index;
+ InstructionPattern::DecodeLoadDoubleWordFromPool(
+ pc - 2 * Instr::kInstrSize, &data_reg, &code_reg, &pool_index);
+ ASSERT(data_reg == R5);
+ ASSERT(code_reg == CODE_REG);
+
+ data_pool_index_ = pool_index;
+ target_pool_index_ = pool_index + 1;
+}
+
NativeCallPattern::NativeCallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
@@ -353,16 +369,6 @@
instr->SetInstructionBits(instr->InstructionBits() | B22);
}
-RawICData* CallPattern::IcData() {
- if (ic_data_.IsNull()) {
- Register reg;
- InstructionPattern::DecodeLoadObject(ic_data_load_end_, object_pool_, ®,
- &ic_data_);
- ASSERT(reg == R5);
- }
- return ic_data_.raw();
-}
-
RawCode* CallPattern::TargetCode() const {
return reinterpret_cast<RawCode*>(
object_pool_.ObjectAt(target_code_pool_index_));
@@ -373,6 +379,24 @@
// No need to flush the instruction cache, since the code is not modified.
}
+RawObject* ICCallPattern::Data() const {
+ return object_pool_.ObjectAt(data_pool_index_);
+}
+
+void ICCallPattern::SetData(const Object& data) const {
+ ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
+ object_pool_.SetObjectAt(data_pool_index_, data);
+}
+
+RawCode* ICCallPattern::TargetCode() const {
+ return reinterpret_cast<RawCode*>(object_pool_.ObjectAt(target_pool_index_));
+}
+
+void ICCallPattern::SetTargetCode(const Code& target) const {
+ object_pool_.SetObjectAt(target_pool_index_, target);
+ // No need to flush the instruction cache, since the code is not modified.
+}
+
SwitchableCallPatternBase::SwitchableCallPatternBase(const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
data_pool_index_(-1),
diff --git a/runtime/vm/instructions_arm64.h b/runtime/vm/instructions_arm64.h
index 8550643..6b8b4aa 100644
--- a/runtime/vm/instructions_arm64.h
+++ b/runtime/vm/instructions_arm64.h
@@ -82,7 +82,23 @@
public:
CallPattern(uword pc, const Code& code);
- RawICData* IcData();
+ RawCode* TargetCode() const;
+ void SetTargetCode(const Code& target) const;
+
+ private:
+ const ObjectPool& object_pool_;
+
+ intptr_t target_code_pool_index_;
+
+ DISALLOW_COPY_AND_ASSIGN(CallPattern);
+};
+
+class ICCallPattern : public ValueObject {
+ public:
+ ICCallPattern(uword pc, const Code& caller_code);
+
+ RawObject* Data() const;
+ void SetData(const Object& data) const;
RawCode* TargetCode() const;
void SetTargetCode(const Code& target) const;
@@ -90,13 +106,10 @@
private:
const ObjectPool& object_pool_;
- uword end_;
- uword ic_data_load_end_;
+ intptr_t target_pool_index_;
+ intptr_t data_pool_index_;
- intptr_t target_code_pool_index_;
- ICData& ic_data_;
-
- DISALLOW_COPY_AND_ASSIGN(CallPattern);
+ DISALLOW_COPY_AND_ASSIGN(ICCallPattern);
};
class NativeCallPattern : public ValueObject {
diff --git a/runtime/vm/instructions_dbc.cc b/runtime/vm/instructions_dbc.cc
index b303e95..6d5280b 100644
--- a/runtime/vm/instructions_dbc.cc
+++ b/runtime/vm/instructions_dbc.cc
@@ -54,15 +54,14 @@
CallPattern::CallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
- ic_data_load_end_(0),
- target_code_pool_index_(-1),
- ic_data_(ICData::Handle()) {
+ data_pool_index_(-1),
+ target_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(end_));
const uword call_pc = end_ - sizeof(Instr);
Instr call_instr = SimulatorBytecode::At(call_pc);
ASSERT(SimulatorBytecode::IsCallOpcode(call_instr));
- ic_data_load_end_ = call_pc;
- target_code_pool_index_ = SimulatorBytecode::DecodeD(call_instr);
+ data_pool_index_ = SimulatorBytecode::DecodeD(call_instr);
+ target_pool_index_ = SimulatorBytecode::DecodeD(call_instr);
}
NativeCallPattern::NativeCallPattern(uword pc, const Code& code)
@@ -143,21 +142,20 @@
return GetLoadedObjectAt(pc, pool, obj);
}
-RawICData* CallPattern::IcData() {
- if (ic_data_.IsNull()) {
- bool found = GetLoadedObjectAt(ic_data_load_end_, object_pool_, &ic_data_);
- ASSERT(found);
- }
- return ic_data_.raw();
+RawObject* CallPattern::Data() const {
+ return object_pool_.ObjectAt(data_pool_index_);
+}
+
+void CallPattern::SetData(const Object& data) const {
+ object_pool_.SetObjectAt(data_pool_index_, data);
}
RawCode* CallPattern::TargetCode() const {
- return reinterpret_cast<RawCode*>(
- object_pool_.ObjectAt(target_code_pool_index_));
+ return reinterpret_cast<RawCode*>(object_pool_.ObjectAt(target_pool_index_));
}
-void CallPattern::SetTargetCode(const Code& target_code) const {
- object_pool_.SetObjectAt(target_code_pool_index_, target_code);
+void CallPattern::SetTargetCode(const Code& target) const {
+ object_pool_.SetObjectAt(target_pool_index_, target);
}
void CallPattern::InsertDeoptCallAt(uword pc) {
diff --git a/runtime/vm/instructions_dbc.h b/runtime/vm/instructions_dbc.h
index 15ac846..3d465fa 100644
--- a/runtime/vm/instructions_dbc.h
+++ b/runtime/vm/instructions_dbc.h
@@ -49,9 +49,10 @@
class CallPattern : public ValueObject {
public:
- CallPattern(uword pc, const Code& code);
+ CallPattern(uword pc, const Code& caller_code);
- RawICData* IcData();
+ RawObject* Data() const;
+ void SetData(const Object& data) const;
RawCode* TargetCode() const;
void SetTargetCode(const Code& code) const;
@@ -62,10 +63,9 @@
const ObjectPool& object_pool_;
uword end_;
- uword ic_data_load_end_;
- intptr_t target_code_pool_index_;
- ICData& ic_data_;
+ intptr_t data_pool_index_;
+ intptr_t target_pool_index_;
DISALLOW_COPY_AND_ASSIGN(CallPattern);
};
diff --git a/runtime/vm/interpreter.cc b/runtime/vm/interpreter.cc
index 2f823c5..c83e454 100644
--- a/runtime/vm/interpreter.cc
+++ b/runtime/vm/interpreter.cc
@@ -565,6 +565,12 @@
USE(entrypoint);
UNIMPLEMENTED();
#elif defined(USING_SIMULATOR)
+ // We need to beware that bouncing between the interpreter and the
+ // simulator may exhaust the C stack before exhausting either the
+ // interpreter or simulator stacks.
+ if (!thread->os_thread()->HasStackHeadroom()) {
+ thread->SetStackLimit(-1);
+ }
result = bit_copy<RawObject*, int64_t>(
Simulator::Current()->Call(reinterpret_cast<intptr_t>(entrypoint),
reinterpret_cast<intptr_t>(code),
diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h
index 914a40f..4813d27 100644
--- a/runtime/vm/isolate.h
+++ b/runtime/vm/isolate.h
@@ -795,6 +795,13 @@
UsingNewBytecodeInstructionsBit::update(value, isolate_flags_);
}
+ bool has_attempted_stepping() const {
+ return HasAttemptedSteppingBit::decode(isolate_flags_);
+ }
+ void set_has_attempted_stepping(bool value) {
+ isolate_flags_ = HasAttemptedSteppingBit::update(value, isolate_flags_);
+ }
+
static void KillAllIsolates(LibMsgId msg_id);
static void KillIfExists(Isolate* isolate, LibMsgId msg_id);
@@ -909,6 +916,7 @@
V(RemappingCids) \
V(ResumeRequest) \
V(HasAttemptedReload) \
+ V(HasAttemptedStepping) \
V(ShouldPausePostServiceRequest) \
V(EnableTypeChecks) \
V(EnableAsserts) \
diff --git a/runtime/vm/isolate_reload.cc b/runtime/vm/isolate_reload.cc
index b153644..75273e8 100644
--- a/runtime/vm/isolate_reload.cc
+++ b/runtime/vm/isolate_reload.cc
@@ -1907,8 +1907,10 @@
function = code.function();
code = function.unoptimized_code();
ASSERT(!code.IsNull());
+ code.ResetSwitchableCalls(zone);
code.ResetICDatas(zone);
} else {
+ code.ResetSwitchableCalls(zone);
code.ResetICDatas(zone);
}
}
@@ -2029,6 +2031,7 @@
if (!stub_code) {
// We are preserving the unoptimized code, fill all ICData arrays with
// the sentinel values so that we have no stale type feedback.
+ code.ResetSwitchableCalls(zone);
code.ResetICDatas(zone);
}
if (!bytecode.IsNull()) {
diff --git a/runtime/vm/native_arguments.h b/runtime/vm/native_arguments.h
index 7c637b3..ab948de 100644
--- a/runtime/vm/native_arguments.h
+++ b/runtime/vm/native_arguments.h
@@ -101,6 +101,14 @@
return *arg_ptr;
}
+ void SetArgAt(int index, const Object& value) const {
+ ASSERT(thread_->execution_state() == Thread::kThreadInVM);
+ ASSERT((index >= 0) && (index < ArgCount()));
+ RawObject** arg_ptr =
+ &(argv_[ReverseArgOrderBit::decode(argc_tag_) ? index : -index]);
+ *arg_ptr = value.raw();
+ }
+
// Does not include hidden type arguments vector.
int NativeArgCount() const {
int function_bits = FunctionBits::decode(argc_tag_);
@@ -158,8 +166,6 @@
return type_args.TypeAt(index);
}
- RawObject** ReturnValueAddress() const { return retval_; }
-
void SetReturn(const Object& value) const {
ASSERT(thread_->execution_state() == Thread::kThreadInVM);
*retval_ = value.raw();
diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc
index 6066a17..fd8cce0 100644
--- a/runtime/vm/object.cc
+++ b/runtime/vm/object.cc
@@ -8006,6 +8006,18 @@
set_ic_data_array(Array::null_array());
}
+RawICData* Function::FindICData(intptr_t deopt_id) const {
+ const Array& array = Array::Handle(ic_data_array());
+ ICData& ic_data = ICData::Handle();
+ for (intptr_t i = 1; i < array.Length(); i++) {
+ ic_data ^= array.At(i);
+ if (ic_data.deopt_id() == deopt_id) {
+ return ic_data.raw();
+ }
+ }
+ return ICData::null();
+}
+
void Function::SetDeoptReasonForAll(intptr_t deopt_id,
ICData::DeoptReasonId reason) {
const Array& array = Array::Handle(ic_data_array());
@@ -8049,6 +8061,10 @@
const Object& result =
Object::Handle(zone, Compiler::CompileFunction(thread, *this));
if (result.IsError()) {
+ if (result.IsLanguageError()) {
+ Exceptions::ThrowCompileTimeError(LanguageError::Cast(result));
+ UNREACHABLE();
+ }
Exceptions::PropagateError(Error::Cast(result));
UNREACHABLE();
}
@@ -14121,19 +14137,10 @@
return result.raw();
}
-bool ICData::AllTargetsHaveSameOwner(intptr_t owner_cid) const {
- if (NumberOfChecksIs(0)) return false;
- Class& cls = Class::Handle();
- const intptr_t len = NumberOfChecks();
- for (intptr_t i = 0; i < len; i++) {
- if (IsUsedAt(i)) {
- cls = Function::Handle(GetTargetAt(i)).Owner();
- if (cls.id() != owner_cid) {
- return false;
- }
- }
- }
- return true;
+RawMegamorphicCache* ICData::AsMegamorphicCache() const {
+ const String& name = String::Handle(target_name());
+ const Array& descriptor = Array::Handle(arguments_descriptor());
+ return MegamorphicCacheTable::Lookup(Isolate::Current(), name, descriptor);
}
bool ICData::HasReceiverClassId(intptr_t class_id) const {
@@ -14152,6 +14159,8 @@
// Returns true if all targets are the same.
// TODO(srdjan): if targets are native use their C_function to compare.
+// TODO(rmacnak): this question should only be asked against a CallTargets,
+// not an ICData.
bool ICData::HasOneTarget() const {
ASSERT(!NumberOfChecksIs(0));
const Function& first_target = Function::Handle(GetTargetAt(0));
@@ -14161,6 +14170,23 @@
return false;
}
}
+ if (is_megamorphic()) {
+ const MegamorphicCache& cache =
+ MegamorphicCache::Handle(AsMegamorphicCache());
+ SafepointMutexLocker ml(Isolate::Current()->megamorphic_lookup_mutex());
+ MegamorphicCacheEntries entries(Array::Handle(cache.buckets()));
+ for (intptr_t i = 0; i < entries.Length(); i++) {
+ const intptr_t id =
+ Smi::Value(entries[i].Get<MegamorphicCache::kClassIdIndex>());
+ if (id == kIllegalCid) {
+ continue;
+ }
+ if (entries[i].Get<MegamorphicCache::kTargetFunctionIndex>() !=
+ first_target.raw()) {
+ return false;
+ }
+ }
+ }
return true;
}
@@ -14319,6 +14345,7 @@
AbstractType::Handle(from.receivers_static_type())));
// Copy deoptimization reasons.
result.SetDeoptReasons(from.DeoptReasons());
+ result.set_is_megamorphic(from.is_megamorphic());
return result.raw();
}
@@ -14343,6 +14370,7 @@
result.set_entries(cloned_array);
// Copy deoptimization reasons.
result.SetDeoptReasons(from.DeoptReasons());
+ result.set_is_megamorphic(from.is_megamorphic());
return result.raw();
}
#endif
@@ -15675,6 +15703,7 @@
void MegamorphicCache::Insert(const Smi& class_id, const Object& target) const {
ASSERT(static_cast<double>(filled_entry_count() + 1) <=
(kLoadFactor * static_cast<double>(mask() + 1)));
+ SafepointMutexLocker ml(Isolate::Current()->megamorphic_lookup_mutex());
const Array& backing_array = Array::Handle(buckets());
intptr_t id_mask = mask();
intptr_t index = (class_id.Value() * kSpreadFactor) & id_mask;
diff --git a/runtime/vm/object.h b/runtime/vm/object.h
index 82b27a0..c1cc1a4 100644
--- a/runtime/vm/object.h
+++ b/runtime/vm/object.h
@@ -1697,6 +1697,18 @@
RebindRule rebind_rule() const;
void set_rebind_rule(uint32_t rebind_rule) const;
+ // This bit is set when a call site becomes megamorphic and starts using a
+ // MegamorphicCache instead of ICData. It means that the entries in the
+ // ICData are incomplete and the MegamorphicCache needs to also be consulted
+ // to list the call site's observed receiver classes and targets.
+ bool is_megamorphic() const {
+ return MegamorphicBit::decode(raw_ptr()->state_bits_);
+ }
+ void set_is_megamorphic(bool value) const {
+ StoreNonPointer(&raw_ptr()->state_bits_,
+ MegamorphicBit::update(value, raw_ptr()->state_bits_));
+ }
+
// The length of the array. This includes all sentinel entries including
// the final one.
intptr_t Length() const;
@@ -1826,9 +1838,9 @@
// Used for printing and optimizations.
RawICData* AsUnaryClassChecksSortedByCount() const;
+ RawMegamorphicCache* AsMegamorphicCache() const;
+
// Consider only used entries.
- bool AllTargetsHaveSameOwner(intptr_t owner_cid) const;
- bool AllReceiversAreNumbers() const;
bool HasOneTarget() const;
bool HasReceiverClassId(intptr_t class_id) const;
@@ -1889,13 +1901,13 @@
intptr_t FindCheck(const GrowableArray<intptr_t>& cids) const;
- private:
- static RawICData* New();
-
RawArray* entries() const {
return AtomicOperations::LoadAcquire(&raw_ptr()->entries_);
}
+ private:
+ static RawICData* New();
+
// Grows the array and also sets the argument to the index that should be used
// for the new entry.
RawArray* Grow(intptr_t* index) const;
@@ -1918,7 +1930,9 @@
kDeoptReasonPos = kTrackingExactnessPos + kTrackingExactnessSize,
kDeoptReasonSize = kLastRecordedDeoptReason + 1,
kRebindRulePos = kDeoptReasonPos + kDeoptReasonSize,
- kRebindRuleSize = 3
+ kRebindRuleSize = 3,
+ kMegamorphicPos = kRebindRulePos + kRebindRuleSize,
+ kMegamorphicSize = 1,
};
COMPILE_ASSERT(kNumRebindRules <= (1 << kRebindRuleSize));
@@ -1939,6 +1953,9 @@
uint32_t,
ICData::kRebindRulePos,
ICData::kRebindRuleSize> {};
+ class MegamorphicBit
+ : public BitField<uint32_t, bool, kMegamorphicPos, kMegamorphicSize> {};
+
#if defined(DEBUG)
// Used in asserts to verify that a check is not added twice.
bool HasCheck(const GrowableArray<intptr_t>& cids) const;
@@ -2863,6 +2880,7 @@
RawArray* ic_data_array() const;
void ClearICDataArray() const;
+ RawICData* FindICData(intptr_t deopt_id) const;
// Sets deopt reason in all ICData-s with given deopt_id.
void SetDeoptReasonForAll(intptr_t deopt_id, ICData::DeoptReasonId reason);
@@ -4488,20 +4506,30 @@
// Note: We keep the checked entrypoint offsets even (emitting NOPs if
// necessary) to allow them to be seen as Smis by the GC.
#if defined(TARGET_ARCH_IA32)
- static const intptr_t kPolymorphicEntryOffset = 0;
- static const intptr_t kMonomorphicEntryOffset = 0;
+ static const intptr_t kMonomorphicEntryOffsetJIT = 6;
+ static const intptr_t kPolymorphicEntryOffsetJIT = 34;
+ static const intptr_t kMonomorphicEntryOffsetAOT = 0;
+ static const intptr_t kPolymorphicEntryOffsetAOT = 0;
#elif defined(TARGET_ARCH_X64)
- static const intptr_t kPolymorphicEntryOffset = 8;
- static const intptr_t kMonomorphicEntryOffset = 32;
+ static const intptr_t kMonomorphicEntryOffsetJIT = 8;
+ static const intptr_t kPolymorphicEntryOffsetJIT = 40;
+ static const intptr_t kMonomorphicEntryOffsetAOT = 8;
+ static const intptr_t kPolymorphicEntryOffsetAOT = 32;
#elif defined(TARGET_ARCH_ARM)
- static const intptr_t kPolymorphicEntryOffset = 0;
- static const intptr_t kMonomorphicEntryOffset = 20;
+ static const intptr_t kMonomorphicEntryOffsetJIT = 0;
+ static const intptr_t kPolymorphicEntryOffsetJIT = 40;
+ static const intptr_t kMonomorphicEntryOffsetAOT = 0;
+ static const intptr_t kPolymorphicEntryOffsetAOT = 20;
#elif defined(TARGET_ARCH_ARM64)
- static const intptr_t kPolymorphicEntryOffset = 8;
- static const intptr_t kMonomorphicEntryOffset = 28;
+ static const intptr_t kMonomorphicEntryOffsetJIT = 8;
+ static const intptr_t kPolymorphicEntryOffsetJIT = 48;
+ static const intptr_t kMonomorphicEntryOffsetAOT = 8;
+ static const intptr_t kPolymorphicEntryOffsetAOT = 28;
#elif defined(TARGET_ARCH_DBC)
- static const intptr_t kPolymorphicEntryOffset = 0;
- static const intptr_t kMonomorphicEntryOffset = 0;
+ static const intptr_t kMonomorphicEntryOffsetJIT = 0;
+ static const intptr_t kPolymorphicEntryOffsetJIT = 0;
+ static const intptr_t kMonomorphicEntryOffsetAOT = 0;
+ static const intptr_t kPolymorphicEntryOffsetAOT = 0;
#else
#error Missing entry offsets for current architecture
#endif
@@ -4509,7 +4537,8 @@
static uword MonomorphicEntryPoint(const RawInstructions* instr) {
uword entry = PayloadStart(instr);
if (!HasSingleEntryPoint(instr)) {
- entry += kPolymorphicEntryOffset;
+ entry += !FLAG_precompiled_mode ? kMonomorphicEntryOffsetJIT
+ : kMonomorphicEntryOffsetAOT;
}
return entry;
}
@@ -4517,7 +4546,8 @@
static uword EntryPoint(const RawInstructions* instr) {
uword entry = PayloadStart(instr);
if (!HasSingleEntryPoint(instr)) {
- entry += kMonomorphicEntryOffset;
+ entry += !FLAG_precompiled_mode ? kPolymorphicEntryOffsetJIT
+ : kPolymorphicEntryOffsetAOT;
}
return entry;
}
@@ -4526,7 +4556,8 @@
uword entry =
PayloadStart(instr) + instr->ptr()->unchecked_entrypoint_pc_offset_;
if (!HasSingleEntryPoint(instr)) {
- entry += kMonomorphicEntryOffset;
+ entry += !FLAG_precompiled_mode ? kPolymorphicEntryOffsetJIT
+ : kPolymorphicEntryOffsetAOT;
}
return entry;
}
@@ -4535,7 +4566,8 @@
uword entry =
PayloadStart(instr) + instr->ptr()->unchecked_entrypoint_pc_offset_;
if (!HasSingleEntryPoint(instr)) {
- entry += kPolymorphicEntryOffset;
+ entry += !FLAG_precompiled_mode ? kMonomorphicEntryOffsetJIT
+ : kMonomorphicEntryOffsetAOT;
}
return entry;
}
@@ -5038,6 +5070,10 @@
return OptimizedBit::decode(raw_ptr()->state_bits_);
}
void set_is_optimized(bool value) const;
+ static bool IsOptimized(RawCode* code) {
+ return Code::OptimizedBit::decode(code->ptr()->state_bits_);
+ }
+
bool is_alive() const { return AliveBit::decode(raw_ptr()->state_bits_); }
void set_is_alive(bool value) const;
@@ -5090,6 +5126,7 @@
// Used during reloading (see object_reload.cc). Calls Reset on all ICDatas
// that are embedded inside the Code or ObjecPool objects.
void ResetICDatas(Zone* zone) const;
+ void ResetSwitchableCalls(Zone* zone) const;
// Array of DeoptInfo objects.
RawArray* deopt_info_array() const {
@@ -5417,10 +5454,6 @@
DISALLOW_COPY_AND_ASSIGN(SlowFindRawCodeVisitor);
};
- static bool IsOptimized(RawCode* code) {
- return Code::OptimizedBit::decode(code->ptr()->state_bits_);
- }
-
static const intptr_t kEntrySize = sizeof(int32_t); // NOLINT
void set_compile_timestamp(int64_t timestamp) const {
@@ -5771,6 +5804,12 @@
static const intptr_t kSpreadFactor = 7;
static const double kLoadFactor;
+ enum EntryType {
+ kClassIdIndex,
+ kTargetFunctionIndex,
+ kEntryLength,
+ };
+
RawArray* buckets() const;
void set_buckets(const Array& buckets) const;
@@ -5817,12 +5856,6 @@
void set_target_name(const String& value) const;
void set_arguments_descriptor(const Array& value) const;
- enum {
- kClassIdIndex,
- kTargetFunctionIndex,
- kEntryLength,
- };
-
static inline void SetEntry(const Array& array,
intptr_t index,
const Smi& class_id,
@@ -9960,6 +9993,9 @@
TypeArguments,
TypeArguments>>;
+using MegamorphicCacheEntries =
+ ArrayOfTuplesView<MegamorphicCache::EntryType, std::tuple<Smi, Object>>;
+
void DumpTypeTable(Isolate* isolate);
void DumpTypeArgumentsTable(Isolate* isolate);
diff --git a/runtime/vm/object_reload.cc b/runtime/vm/object_reload.cc
index f9eb462..032fb60 100644
--- a/runtime/vm/object_reload.cc
+++ b/runtime/vm/object_reload.cc
@@ -4,10 +4,13 @@
#include "vm/object.h"
+#include "vm/code_patcher.h"
#include "vm/hash_table.h"
#include "vm/isolate_reload.h"
#include "vm/log.h"
+#include "vm/object_store.h"
#include "vm/resolver.h"
+#include "vm/stub_code.h"
#include "vm/symbols.h"
namespace dart {
@@ -89,6 +92,63 @@
#endif
}
+void Code::ResetSwitchableCalls(Zone* zone) const {
+#if !defined(TARGET_ARCH_DBC)
+ if (is_optimized()) {
+ return; // No switchable calls in optimized code.
+ }
+
+ const Object& owner = Object::Handle(zone, this->owner());
+ if (!owner.IsFunction()) {
+ return; // No switchable calls in stub code.
+ }
+
+ const Array& ic_data_array =
+ Array::Handle(zone, Function::Cast(owner).ic_data_array());
+ if (ic_data_array.IsNull()) {
+ // The megamorphic miss stub and some recognized function doesn't populate
+ // their ic_data_array. Check this only happens for functions without IC
+ // calls.
+#if defined(DEBUG)
+ const PcDescriptors& descriptors =
+ PcDescriptors::Handle(zone, pc_descriptors());
+ PcDescriptors::Iterator iter(descriptors, RawPcDescriptors::kIcCall);
+ while (iter.MoveNext()) {
+ FATAL1("%s has IC calls but no ic_data_array\n", owner.ToCString());
+ }
+#endif
+ return;
+ }
+ ICData& ic_data = ICData::Handle(zone);
+ Object& data = Object::Handle(zone);
+ for (intptr_t i = 1; i < ic_data_array.Length(); i++) {
+ ic_data ^= ic_data_array.At(i);
+ if (ic_data.rebind_rule() != ICData::kInstance) {
+ continue;
+ }
+ if (ic_data.NumArgsTested() != 1) {
+ continue;
+ }
+ uword pc = GetPcForDeoptId(ic_data.deopt_id(), RawPcDescriptors::kIcCall);
+ CodePatcher::GetInstanceCallAt(pc, *this, &data);
+ // This check both avoids unnecessary patching to reduce log spam and
+ // prevents patching over breakpoint stubs.
+ if (!data.IsICData()) {
+ const Code& stub =
+ ic_data.is_tracking_exactness()
+ ? StubCode::OneArgCheckInlineCacheWithExactnessCheck()
+ : StubCode::OneArgCheckInlineCache();
+ CodePatcher::PatchInstanceCallAt(pc, *this, ic_data, stub);
+ if (FLAG_trace_ic) {
+ OS::PrintErr("Instance call at %" Px
+ " resetting to polymorphic dispatch, %s\n",
+ pc, ic_data.ToCString());
+ }
+ }
+ }
+#endif
+}
+
void Bytecode::ResetICDatas(Zone* zone) const {
// Iterate over the Bytecode's object pool and reset all ICDatas.
const ObjectPool& pool = ObjectPool::Handle(zone, object_pool());
diff --git a/runtime/vm/runtime_entry.cc b/runtime/vm/runtime_entry.cc
index e997516..c2257d0 100644
--- a/runtime/vm/runtime_entry.cc
+++ b/runtime/vm/runtime_entry.cc
@@ -1075,6 +1075,87 @@
return result.raw();
}
+static void TrySwitchInstanceCall(const ICData& ic_data,
+ const Function& target_function) {
+#if !defined(TARGET_ARCH_DBC) && !defined(DART_PRECOMPILED_RUNTIME)
+ // Monomorphic/megamorphic calls only check the receiver CID.
+ if (ic_data.NumArgsTested() != 1) return;
+
+ ASSERT(ic_data.rebind_rule() == ICData::kInstance);
+
+ // Monomorphic/megamorphic calls don't record exactness.
+ if (ic_data.is_tracking_exactness()) return;
+
+#if !defined(PRODUCT)
+ // Monomorphic/megamorphic do not check the isolate's stepping flag.
+ if (Isolate::Current()->has_attempted_stepping()) return;
+#endif
+
+ Thread* thread = Thread::Current();
+ DartFrameIterator iterator(thread,
+ StackFrameIterator::kNoCrossThreadIteration);
+ StackFrame* caller_frame = iterator.NextFrame();
+ ASSERT(caller_frame->IsDartFrame());
+
+ // Monomorphic/megamorphic calls are only for unoptimized code.
+ if (caller_frame->is_interpreted()) return;
+ Zone* zone = thread->zone();
+ const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
+ if (caller_code.is_optimized()) return;
+
+ // Code is detached from its function. This will prevent us from resetting
+ // the switchable call later because resets are function based and because
+ // the ic_data_array belongs to the function instead of the code. This should
+ // only happen because of reload, but it sometimes happens with KBC mixed mode
+ // probably through a race between foreground and background compilation.
+ const Function& caller_function =
+ Function::Handle(zone, caller_code.function());
+ if (caller_function.unoptimized_code() != caller_code.raw()) {
+ return;
+ }
+
+ intptr_t num_checks = ic_data.NumberOfChecks();
+
+ // Monomorphic call.
+ if (num_checks == 1) {
+ // A call site in the monomorphic state does not load the arguments
+ // descriptor, so do not allow transition to this state if the callee
+ // needs it.
+ if (target_function.HasOptionalParameters() ||
+ target_function.IsGeneric()) {
+ return;
+ }
+
+ const Array& data = Array::Handle(zone, ic_data.entries());
+ const Code& target = Code::Handle(zone, target_function.EnsureHasCode());
+ CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, data,
+ target);
+ if (FLAG_trace_ic) {
+ OS::PrintErr("Instance call at %" Px
+ " switching to monomorphic dispatch, %s\n",
+ caller_frame->pc(), ic_data.ToCString());
+ }
+ return; // Success.
+ }
+
+ // Megamorphic call.
+ if (num_checks > FLAG_max_polymorphic_checks) {
+ const MegamorphicCache& cache =
+ MegamorphicCache::Handle(zone, ic_data.AsMegamorphicCache());
+ ic_data.set_is_megamorphic(true);
+ CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, cache,
+ StubCode::MegamorphicCall());
+ if (FLAG_trace_ic) {
+ OS::PrintErr("Instance call at %" Px
+ " switching to megamorphic dispatch, %s\n",
+ caller_frame->pc(), ic_data.ToCString());
+ }
+ return; // Success.
+ }
+
+#endif // !defined(TARGET_ARCH_DBC) && !defined(DART_PRECOMPILED_RUNTIME)
+}
+
// Perform the subtype and return constant function based on the result.
static RawFunction* ComputeTypeCheckTarget(const Instance& receiver,
const AbstractType& type,
@@ -1091,7 +1172,8 @@
static RawFunction* InlineCacheMissHandler(
const GrowableArray<const Instance*>& args, // Checked arguments only.
- const ICData& ic_data) {
+ const ICData& ic_data,
+ intptr_t count = 1) {
const Instance& receiver = *args[0];
ArgumentsDescriptor arguments_descriptor(
Array::Handle(ic_data.arguments_descriptor()));
@@ -1136,13 +1218,13 @@
ic_data.receivers_static_type())),
receiver);
ic_data.AddReceiverCheck(
- receiver.GetClassId(), target_function,
- /*count=*/1, /*exactness=*/state.CollapseSuperTypeExactness());
+ receiver.GetClassId(), target_function, count,
+ /*exactness=*/state.CollapseSuperTypeExactness());
#else
UNREACHABLE();
#endif
} else {
- ic_data.AddReceiverCheck(args[0]->GetClassId(), target_function);
+ ic_data.AddReceiverCheck(args[0]->GetClassId(), target_function, count);
}
} else {
GrowableArray<intptr_t> class_ids(args.length());
@@ -1150,7 +1232,7 @@
for (intptr_t i = 0; i < args.length(); i++) {
class_ids.Add(args[i]->GetClassId());
}
- ic_data.AddCheck(class_ids, target_function);
+ ic_data.AddCheck(class_ids, target_function, count);
}
if (FLAG_trace_ic_miss_in_optimized || FLAG_trace_ic) {
DartFrameIterator iterator(Thread::Current(),
@@ -1174,6 +1256,9 @@
receiver.GetClassId(), target_function.ToCString());
}
}
+
+ TrySwitchInstanceCall(ic_data, target_function);
+
return target_function.raw();
}
@@ -1289,9 +1374,10 @@
#endif
// Handle a miss of a single target cache.
-// Arg0: Receiver.
+// Arg1: Receiver.
+// Arg0: Stub out.
// Returns: the ICData used to continue with a polymorphic call.
-DEFINE_RUNTIME_ENTRY(SingleTargetMiss, 1) {
+DEFINE_RUNTIME_ENTRY(SingleTargetMiss, 2) {
#if defined(TARGET_ARCH_DBC)
// DBC does not use switchable calls.
UNREACHABLE();
@@ -1360,6 +1446,7 @@
cache.set_upper_limit(upper);
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
+ arguments.SetArgAt(0, StubCode::ICCallThroughCode());
arguments.SetReturn(ic_data);
return;
}
@@ -1373,18 +1460,24 @@
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
+ arguments.SetArgAt(0, stub);
arguments.SetReturn(ic_data);
#endif
}
-DEFINE_RUNTIME_ENTRY(UnlinkedCall, 2) {
+// Handle the first use of an instance call
+// Arg2: UnlinkedCall.
+// Arg1: Receiver.
+// Arg0: Stub out.
+// Returns: the ICData used to continue with a polymorphic call.
+DEFINE_RUNTIME_ENTRY(UnlinkedCall, 3) {
#if defined(TARGET_ARCH_DBC)
// DBC does not use switchable calls.
UNREACHABLE();
#else
- const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
+ const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
const UnlinkedCall& unlinked =
- UnlinkedCall::CheckedHandle(zone, arguments.ArgAt(1));
+ UnlinkedCall::CheckedHandle(zone, arguments.ArgAt(2));
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
@@ -1426,6 +1519,7 @@
// Return the ICData. The miss stub will jump to continue in the IC call
// stub.
+ arguments.SetArgAt(0, StubCode::ICCallThroughCode());
arguments.SetReturn(ic_data);
return;
}
@@ -1438,19 +1532,41 @@
// Return the ICData. The miss stub will jump to continue in the IC lookup
// stub.
+ arguments.SetArgAt(0, stub);
arguments.SetReturn(ic_data);
#endif // !DBC
}
+#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC)
+static RawICData* FindICDataForInstanceCall(Zone* zone,
+ const Code& code,
+ uword pc) {
+ uword pc_offset = pc - code.PayloadStart();
+ const PcDescriptors& descriptors =
+ PcDescriptors::Handle(zone, code.pc_descriptors());
+ PcDescriptors::Iterator iter(descriptors, RawPcDescriptors::kIcCall);
+ intptr_t deopt_id = -1;
+ while (iter.MoveNext()) {
+ if (iter.PcOffset() == pc_offset) {
+ deopt_id = iter.DeoptId();
+ break;
+ }
+ }
+ ASSERT(deopt_id != -1);
+ return Function::Handle(zone, code.function()).FindICData(deopt_id);
+}
+#endif // !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC)
+
// Handle a miss of a megamorphic cache.
-// Arg0: Receiver.
+// Arg1: Receiver.
+// Arg0: continuation Code (out parameter).
// Returns: the ICData used to continue with a polymorphic call.
-DEFINE_RUNTIME_ENTRY(MonomorphicMiss, 1) {
+DEFINE_RUNTIME_ENTRY(MonomorphicMiss, 2) {
#if defined(TARGET_ARCH_DBC)
// DBC does not use switchable calls.
UNREACHABLE();
-#else
- const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
+#elif defined(DART_PRECOMPILED_RUNTIME)
+ const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
@@ -1520,6 +1636,7 @@
stub);
// Return the ICData. The miss stub will jump to continue in the IC call
// stub.
+ arguments.SetArgAt(0, StubCode::ICCallThroughCode());
arguments.SetReturn(ic_data);
return;
}
@@ -1533,6 +1650,46 @@
// Return the ICData. The miss stub will jump to continue in the IC lookup
// stub.
+ arguments.SetArgAt(0, stub);
+ arguments.SetReturn(ic_data);
+#else // JIT
+ DartFrameIterator iterator(thread,
+ StackFrameIterator::kNoCrossThreadIteration);
+ StackFrame* caller_frame = iterator.NextFrame();
+ ASSERT(caller_frame->IsDartFrame());
+ ASSERT(!caller_frame->is_interpreted());
+ const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
+ ASSERT(!caller_code.is_optimized());
+
+ const ICData& ic_data = ICData::Handle(
+ zone, FindICDataForInstanceCall(zone, caller_code, caller_frame->pc()));
+ RELEASE_ASSERT(!ic_data.IsNull());
+
+ ASSERT(ic_data.NumArgsTested() == 1);
+ const Code& stub = ic_data.is_tracking_exactness()
+ ? StubCode::OneArgCheckInlineCacheWithExactnessCheck()
+ : StubCode::OneArgCheckInlineCache();
+ CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, ic_data,
+ stub);
+ if (FLAG_trace_ic) {
+ OS::PrintErr("Instance call at %" Px
+ " switching to polymorphic dispatch, %s\n",
+ caller_frame->pc(), ic_data.ToCString());
+ }
+
+ const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
+ // ICData can be shared between unoptimized and optimized code, so beware that
+ // the new receiver class may have already been added through the optimized
+ // code.
+ if (!ic_data.HasReceiverClassId(receiver.GetClassId())) {
+ GrowableArray<const Instance*> args(1);
+ args.Add(&receiver);
+ // Don't count during insertion because the IC stub we continue through will
+ // do an increment.
+ intptr_t count = 0;
+ InlineCacheMissHandler(args, ic_data, count);
+ }
+ arguments.SetArgAt(0, stub);
arguments.SetReturn(ic_data);
#endif // !defined(TARGET_ARCH_DBC)
}
@@ -2077,7 +2234,7 @@
// process the stack overflow now and leave the interrupt for next
// time.
// TODO(regis): Warning: IsCalleeFrameOf is overridden in stack_frame_dbc.h.
- if (interpreter_stack_overflow ||
+ if (interpreter_stack_overflow || !thread->os_thread()->HasStackHeadroom() ||
IsCalleeFrameOf(thread->saved_stack_limit(), stack_pos)) {
// Use the preallocated stack overflow exception to avoid calling
// into dart code.
@@ -2217,7 +2374,7 @@
}
ASSERT(frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
- ASSERT(caller_code.is_optimized());
+ RELEASE_ASSERT(caller_code.is_optimized());
const Function& target_function = Function::Handle(
zone, caller_code.GetStaticCallTargetFunctionAt(frame->pc()));
@@ -2241,6 +2398,52 @@
#endif
}
+// The caller must be a monomorphic call from unoptimized code.
+// Patch call to point to new target.
+DEFINE_RUNTIME_ENTRY(FixCallersTargetMonomorphic, 0) {
+#if !defined(DART_PRECOMPILED_RUNTIME)
+ StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
+ StackFrameIterator::kNoCrossThreadIteration);
+ StackFrame* frame = iterator.NextFrame();
+ ASSERT(frame != NULL);
+ while (frame->IsStubFrame() || frame->IsExitFrame()) {
+ frame = iterator.NextFrame();
+ ASSERT(frame != NULL);
+ }
+ if (frame->IsEntryFrame()) {
+ // Since function's current code is always unpatched, the entry frame always
+ // calls to unpatched code.
+ UNREACHABLE();
+ }
+ ASSERT(frame->IsDartFrame());
+ const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
+ RELEASE_ASSERT(!caller_code.is_optimized());
+
+ Object& cache = Object::Handle(zone);
+ const Code& old_target_code = Code::Handle(
+ zone, CodePatcher::GetInstanceCallAt(frame->pc(), caller_code, &cache));
+ const Function& target_function =
+ Function::Handle(zone, old_target_code.function());
+ const Code& current_target_code =
+ Code::Handle(zone, target_function.EnsureHasCode());
+ CodePatcher::PatchInstanceCallAt(frame->pc(), caller_code, cache,
+ current_target_code);
+ if (FLAG_trace_patching) {
+ OS::PrintErr(
+ "FixCallersTargetMonomorphic: caller %#" Px
+ " "
+ "target '%s' -> %#" Px " (%s)\n",
+ frame->pc(), target_function.ToFullyQualifiedCString(),
+ current_target_code.EntryPoint(),
+ current_target_code.is_optimized() ? "optimized" : "unoptimized");
+ }
+ ASSERT(!current_target_code.IsDisabled());
+ arguments.SetReturn(current_target_code);
+#else
+ UNREACHABLE();
+#endif
+}
+
// The caller tried to allocate an instance via an invalidated allocation
// stub.
DEFINE_RUNTIME_ENTRY(FixAllocationStubTarget, 0) {
diff --git a/runtime/vm/runtime_entry_list.h b/runtime/vm/runtime_entry_list.h
index b0c241e..f5f806b 100644
--- a/runtime/vm/runtime_entry_list.h
+++ b/runtime/vm/runtime_entry_list.h
@@ -18,6 +18,7 @@
V(GetFieldForDispatch) \
V(ResolveCallFunction) \
V(FixCallersTarget) \
+ V(FixCallersTargetMonomorphic) \
V(FixAllocationStubTarget) \
V(InlineCacheMissHandlerOneArg) \
V(InlineCacheMissHandlerTwoArgs) \
diff --git a/runtime/vm/stub_code_list.h b/runtime/vm/stub_code_list.h
index 248c455..f0eff2c 100644
--- a/runtime/vm/stub_code_list.h
+++ b/runtime/vm/stub_code_list.h
@@ -46,6 +46,7 @@
V(UnoptimizedIdenticalWithNumberCheck) \
V(OptimizedIdenticalWithNumberCheck) \
V(ICCallBreakpoint) \
+ V(UnoptStaticCallBreakpoint) \
V(RuntimeCallBreakpoint) \
V(OneArgCheckInlineCache) \
V(TwoArgsCheckInlineCache) \
diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h
index 44bb9e6..ab8071e 100644
--- a/runtime/vm/thread.h
+++ b/runtime/vm/thread.h
@@ -112,8 +112,6 @@
V(RawCode*, stack_overflow_shared_with_fpu_regs_stub_, \
StubCode::StackOverflowSharedWithFPURegs().raw(), NULL) \
V(RawCode*, monomorphic_miss_stub_, StubCode::MonomorphicMiss().raw(), NULL) \
- V(RawCode*, ic_lookup_through_code_stub_, \
- StubCode::ICCallThroughCode().raw(), NULL) \
V(RawCode*, optimize_stub_, StubCode::OptimizeFunction().raw(), NULL) \
V(RawCode*, deoptimize_stub_, StubCode::Deoptimize().raw(), NULL) \
V(RawCode*, lazy_deopt_from_return_stub_, \
