[vm] Re-order ICData entries to allow CID and count to be accessed with a load-pair.
Bug: https://github.com/dart-lang/sdk/issues/36731
Change-Id: Ia91ddb7e8991a5ab227ab8758173a99141c259be
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/103004
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
diff --git a/runtime/vm/compiler/runtime_api.cc b/runtime/vm/compiler/runtime_api.cc
index 8f927b8..758e065 100644
--- a/runtime/vm/compiler/runtime_api.cc
+++ b/runtime/vm/compiler/runtime_api.cc
@@ -368,8 +368,8 @@
return dart::ICData::TargetIndexFor(num_args);
}
-word ICData::ExactnessOffsetFor(word num_args) {
- return dart::ICData::ExactnessOffsetFor(num_args);
+word ICData::ExactnessIndexFor(word num_args) {
+ return dart::ICData::ExactnessIndexFor(num_args);
}
word ICData::TestEntryLengthFor(word num_args, bool exactness_check) {
diff --git a/runtime/vm/compiler/runtime_api.h b/runtime/vm/compiler/runtime_api.h
index d910575..29ff5ba 100644
--- a/runtime/vm/compiler/runtime_api.h
+++ b/runtime/vm/compiler/runtime_api.h
@@ -419,7 +419,7 @@
static word CodeIndexFor(word num_args);
static word CountIndexFor(word num_args);
static word TargetIndexFor(word num_args);
- static word ExactnessOffsetFor(word num_args);
+ static word ExactnessIndexFor(word num_args);
static word TestEntryLengthFor(word num_args, bool exactness_check);
static word EntryPointIndexFor(word num_args);
static word NumArgsTestedShift();
diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc
index 4313cd4..91728ca 100644
--- a/runtime/vm/compiler/stub_code_compiler_x64.cc
+++ b/runtime/vm/compiler/stub_code_compiler_x64.cc
@@ -2036,7 +2036,7 @@
const intptr_t count_offset =
target::ICData::CountIndexFor(num_args) * target::kWordSize;
const intptr_t exactness_offset =
- target::ICData::ExactnessOffsetFor(num_args) * target::kWordSize;
+ target::ICData::ExactnessIndexFor(num_args) * target::kWordSize;
__ Bind(&loop);
for (int unroll = optimize ? 4 : 2; unroll >= 0; unroll--) {
diff --git a/runtime/vm/interpreter.cc b/runtime/vm/interpreter.cc
index 3307d3e..a0ba01e 100644
--- a/runtime/vm/interpreter.cc
+++ b/runtime/vm/interpreter.cc
@@ -793,7 +793,7 @@
intptr_t i;
for (i = 0; i < (length - (kCheckedArgs + 2)); i += (kCheckedArgs + 2)) {
if (cache->data()[i + 0] == receiver_cid) {
- top[0] = cache->data()[i + kCheckedArgs];
+ top[0] = cache->data()[i + ICData::TargetIndexFor(kCheckedArgs)];
found = true;
break;
}
@@ -841,7 +841,7 @@
for (i = 0; i < (length - (kCheckedArgs + 2)); i += (kCheckedArgs + 2)) {
if ((cache->data()[i + 0] == receiver_cid) &&
(cache->data()[i + 1] == arg0_cid)) {
- top[0] = cache->data()[i + kCheckedArgs];
+ top[0] = cache->data()[i + ICData::TargetIndexFor(kCheckedArgs)];
found = true;
break;
}
diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc
index d2e8a61..73cf977 100644
--- a/runtime/vm/object.cc
+++ b/runtime/vm/object.cc
@@ -13474,7 +13474,8 @@
}
// The final entry is always the sentinel.
ASSERT(IsSentinelAt(len - 1));
- if (NumArgsTested() == 0) {
+ const intptr_t num_args_tested = NumArgsTested();
+ if (num_args_tested == 0) {
// No type feedback is being collected.
const Array& data = Array::Handle(entries());
// Static calls with no argument checks hold only one target and the
@@ -13483,11 +13484,11 @@
// Static calls with no argument checks only need two words.
ASSERT(TestEntryLength() == 2);
// Set the target.
- data.SetAt(0, func);
+ data.SetAt(TargetIndexFor(num_args_tested), func);
// Set count to 0 as this is called during compilation, before the
// call has been executed.
const Smi& value = Smi::Handle(Smi::New(0));
- data.SetAt(1, value);
+ data.SetAt(CountIndexFor(num_args_tested), value);
} else {
// Type feedback on arguments is being collected.
const Array& data = Array::Handle(entries());
@@ -13501,9 +13502,9 @@
for (intptr_t i = 0; i < NumArgsTested(); i++) {
data.SetAt(i, object_cid);
}
- data.SetAt(NumArgsTested(), func);
+ data.SetAt(TargetIndexFor(num_args_tested), func);
const Smi& value = Smi::Handle(Smi::New(0));
- data.SetAt(NumArgsTested() + 1, value);
+ data.SetAt(CountIndexFor(num_args_tested), value);
}
}
@@ -13573,11 +13574,11 @@
WriteSentinel(data, TestEntryLength());
intptr_t data_pos = old_num * TestEntryLength();
ASSERT(!target.IsNull());
- data.SetAt(data_pos++, target);
+ data.SetAt(data_pos + TargetIndexFor(NumArgsTested()), target);
// Set count to 0 as this is called during compilation, before the
// call has been executed.
const Smi& value = Smi::Handle(Smi::New(0));
- data.SetAt(data_pos, value);
+ data.SetAt(data_pos + CountIndexFor(NumArgsTested()), value);
// Multithreaded access to ICData requires setting of array to be the last
// operation.
set_entries(data);
@@ -13608,7 +13609,8 @@
ASSERT((target.name() == target_name()) || ValidateInterceptor(target));
DEBUG_ASSERT(!HasCheck(class_ids));
ASSERT(NumArgsTested() > 1); // Otherwise use 'AddReceiverCheck'.
- ASSERT(class_ids.length() == NumArgsTested());
+ const intptr_t num_args_tested = NumArgsTested();
+ ASSERT(class_ids.length() == num_args_tested);
const intptr_t old_num = NumberOfChecks();
Array& data = Array::Handle(entries());
// ICData of static calls with NumArgsTested() > 0 have initially a
@@ -13616,14 +13618,14 @@
// overwritten by first real type feedback data.
if (old_num == 1) {
bool has_dummy_entry = true;
- for (intptr_t i = 0; i < NumArgsTested(); i++) {
+ for (intptr_t i = 0; i < num_args_tested; i++) {
if (Smi::Value(Smi::RawCast(data.At(i))) != kObjectCid) {
has_dummy_entry = false;
break;
}
}
if (has_dummy_entry) {
- ASSERT(target.raw() == data.At(NumArgsTested()));
+ ASSERT(target.raw() == data.At(TargetIndexFor(num_args_tested)));
// Replace dummy entry.
Smi& value = Smi::Handle();
for (intptr_t i = 0; i < NumArgsTested(); i++) {
@@ -13643,12 +13645,12 @@
// kIllegalCid is used as terminating value, do not add it.
ASSERT(class_ids[i] != kIllegalCid);
value = Smi::New(class_ids[i]);
- data.SetAt(data_pos++, value);
+ data.SetAt(data_pos + i, value);
}
ASSERT(!target.IsNull());
- data.SetAt(data_pos++, target);
+ data.SetAt(data_pos + TargetIndexFor(num_args_tested), target);
value = Smi::New(count);
- data.SetAt(data_pos++, value);
+ data.SetAt(data_pos + CountIndexFor(num_args_tested), value);
// Multithreaded access to ICData requires setting of array to be the last
// operation.
set_entries(data);
@@ -13697,7 +13699,8 @@
ASSERT(!HasCheck(class_ids));
#endif // DEBUG
ASSERT(!target.IsNull());
- ASSERT(NumArgsTested() == 1); // Otherwise use 'AddCheck'.
+ const intptr_t kNumArgsTested = 1;
+ ASSERT(NumArgsTested() == kNumArgsTested); // Otherwise use 'AddCheck'.
ASSERT(receiver_class_id != kIllegalCid);
intptr_t index = -1;
@@ -13714,10 +13717,12 @@
}
data.SetAt(data_pos, Smi::Handle(Smi::New(receiver_class_id)));
if (Isolate::Current()->compilation_allowed()) {
- data.SetAt(data_pos + 1, target);
- data.SetAt(data_pos + 2, Smi::Handle(Smi::New(count)));
+ data.SetAt(data_pos + TargetIndexFor(kNumArgsTested), target);
+ data.SetAt(data_pos + CountIndexFor(kNumArgsTested),
+ Smi::Handle(Smi::New(count)));
if (is_tracking_exactness()) {
- data.SetAt(data_pos + 3, Smi::Handle(Smi::New(exactness.Encode())));
+ data.SetAt(data_pos + ExactnessIndexFor(kNumArgsTested),
+ Smi::Handle(Smi::New(exactness.Encode())));
}
} else {
// Precompilation only, after all functions have been compiled.
@@ -13725,8 +13730,8 @@
const Code& code = Code::Handle(target.CurrentCode());
const Smi& entry_point =
Smi::Handle(Smi::FromAlignedAddress(code.EntryPoint()));
- data.SetAt(data_pos + 1, code);
- data.SetAt(data_pos + 2, entry_point);
+ data.SetAt(data_pos + CodeIndexFor(kNumArgsTested), code);
+ data.SetAt(data_pos + EntryPointIndexFor(kNumArgsTested), entry_point);
}
// Multithreaded access to ICData requires setting of array to be the last
// operation.
@@ -13738,9 +13743,10 @@
return StaticTypeExactnessState::NotTracking();
}
const Array& data = Array::Handle(entries());
- intptr_t data_pos = index * TestEntryLength();
+ intptr_t data_pos =
+ index * TestEntryLength() + ExactnessIndexFor(NumArgsTested());
return StaticTypeExactnessState::Decode(
- Smi::Value(Smi::RawCast(data.At(data_pos + NumArgsTested() + 2))));
+ Smi::Value(Smi::RawCast(data.At(data_pos))));
}
void ICData::GetCheckAt(intptr_t index,
@@ -13753,9 +13759,9 @@
const Array& data = Array::Handle(entries());
intptr_t data_pos = index * TestEntryLength();
for (intptr_t i = 0; i < NumArgsTested(); i++) {
- class_ids->Add(Smi::Value(Smi::RawCast(data.At(data_pos++))));
+ class_ids->Add(Smi::Value(Smi::RawCast(data.At(data_pos + i))));
}
- (*target) ^= data.At(data_pos++);
+ (*target) ^= data.At(data_pos + TargetIndexFor(NumArgsTested()));
}
bool ICData::IsSentinelAt(intptr_t index) const {
@@ -13794,7 +13800,7 @@
const Array& data = Array::Handle(entries());
const intptr_t data_pos = index * TestEntryLength();
*class_id = Smi::Value(Smi::RawCast(data.At(data_pos)));
- *target ^= data.At(data_pos + 1);
+ *target ^= data.At(data_pos + TargetIndexFor(NumArgsTested()));
}
intptr_t ICData::GetCidAt(intptr_t index) const {
@@ -13821,7 +13827,8 @@
RawFunction* ICData::GetTargetAt(intptr_t index) const {
ASSERT(Isolate::Current()->compilation_allowed());
- const intptr_t data_pos = index * TestEntryLength() + NumArgsTested();
+ const intptr_t data_pos =
+ index * TestEntryLength() + TargetIndexFor(NumArgsTested());
ASSERT(Object::Handle(Array::Handle(entries()).At(data_pos)).IsFunction());
NoSafepointScope no_safepoint;
@@ -13830,7 +13837,8 @@
}
RawObject* ICData::GetTargetOrCodeAt(intptr_t index) const {
- const intptr_t data_pos = index * TestEntryLength() + NumArgsTested();
+ const intptr_t data_pos =
+ index * TestEntryLength() + TargetIndexFor(NumArgsTested());
NoSafepointScope no_safepoint;
RawArray* raw_data = entries();
@@ -13894,18 +13902,6 @@
}
#if !defined(DART_PRECOMPILED_RUNTIME)
-RawFunction* ICData::GetTargetForReceiverClassId(intptr_t class_id,
- intptr_t* count_return) const {
- const intptr_t len = NumberOfChecks();
- for (intptr_t i = 0; i < len; i++) {
- if (GetReceiverClassIdAt(i) == class_id) {
- *count_return = GetCountAt(i);
- return GetTargetAt(i);
- }
- }
- return Function::null();
-}
-
RawICData* ICData::AsUnaryClassChecksForCid(intptr_t cid,
const Function& target) const {
ASSERT(!IsNull());
diff --git a/runtime/vm/object.h b/runtime/vm/object.h
index 2b04c83..725bc63 100644
--- a/runtime/vm/object.h
+++ b/runtime/vm/object.h
@@ -1775,8 +1775,6 @@
intptr_t GetClassIdAt(intptr_t index, intptr_t arg_nr) const;
RawFunction* GetTargetAt(intptr_t index) const;
- RawFunction* GetTargetForReceiverClassId(intptr_t class_id,
- intptr_t* count_return) const;
RawObject* GetTargetOrCodeAt(intptr_t index) const;
void SetCodeAt(intptr_t index, const Code& value) const;
@@ -1829,16 +1827,13 @@
static intptr_t TestEntryLengthFor(intptr_t num_args,
bool tracking_exactness);
- static intptr_t TargetIndexFor(intptr_t num_args) { return num_args; }
- static intptr_t CodeIndexFor(intptr_t num_args) { return num_args; }
+ static intptr_t CountIndexFor(intptr_t num_args) { return num_args; }
+ static intptr_t EntryPointIndexFor(intptr_t num_args) { return num_args; }
- static intptr_t CountIndexFor(intptr_t num_args) { return (num_args + 1); }
- static intptr_t EntryPointIndexFor(intptr_t num_args) {
- return (num_args + 1);
- }
- static intptr_t ExactnessOffsetFor(intptr_t num_args) {
- return (num_args + 2);
- }
+ static intptr_t TargetIndexFor(intptr_t num_args) { return num_args + 1; }
+ static intptr_t CodeIndexFor(intptr_t num_args) { return num_args + 1; }
+
+ static intptr_t ExactnessIndexFor(intptr_t num_args) { return num_args + 2; }
bool IsUsedAt(intptr_t i) const;
diff --git a/runtime/vm/simulator_dbc.cc b/runtime/vm/simulator_dbc.cc
index 23fa5e7..21c3382 100644
--- a/runtime/vm/simulator_dbc.cc
+++ b/runtime/vm/simulator_dbc.cc
@@ -874,7 +874,7 @@
intptr_t i;
for (i = 0; i < (length - (kCheckedArgs + 2)); i += (kCheckedArgs + 2)) {
if (cache->data()[i + 0] == receiver_cid) {
- top[0] = cache->data()[i + kCheckedArgs];
+ top[0] = cache->data()[i + ICData::TargetIndexFor(kCheckedArgs)];
found = true;
break;
}
@@ -920,7 +920,7 @@
for (i = 0; i < (length - (kCheckedArgs + 2)); i += (kCheckedArgs + 2)) {
if ((cache->data()[i + 0] == receiver_cid) &&
(cache->data()[i + 1] == arg0_cid)) {
- top[0] = cache->data()[i + kCheckedArgs];
+ top[0] = cache->data()[i + ICData::TargetIndexFor(kCheckedArgs)];
found = true;
break;
}
