[VM/SIMDBC] Add simdbc64 bytecodes to allow LoadIndexed/StoreIndexed to work on untagged arrays
Issue https://github.com/dart-lang/sdk/issues/35154
Change-Id: I86db977ce6c618fbbff6186cd75c8dc84546f6f9
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/97302
Commit-Queue: Martin Kustermann <kustermann@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
diff --git a/runtime/vm/compiler/backend/il_dbc.cc b/runtime/vm/compiler/backend/il_dbc.cc
index 0ffd8d9..580ffee 100644
--- a/runtime/vm/compiler/backend/il_dbc.cc
+++ b/runtime/vm/compiler/backend/il_dbc.cc
@@ -748,7 +748,7 @@
case kExternalTypedDataUint8ArrayCid:
ASSERT(index_scale() == 1);
if (IsExternal()) {
- __ StoreIndexedExternalUint8(array, index, value);
+ __ StoreIndexedUntaggedUint8(array, index, value);
} else {
__ StoreIndexedUint8(array, index, value);
}
@@ -760,43 +760,58 @@
case kTypedDataInt32ArrayCid:
case kTypedDataUint32ArrayCid: {
if (IsExternal()) {
- Unsupported(compiler);
- UNREACHABLE();
- }
- if (index_scale() == 1) {
- __ StoreIndexedUint32(array, index, value);
+ if (index_scale() == 1) {
+ __ StoreIndexedUntaggedUint32(array, index, value);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ StoreIndexedUntaggedUint32(array, temp, value);
+ }
} else {
- __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
- __ StoreIndexedUint32(array, temp, value);
+ if (index_scale() == 1) {
+ __ StoreIndexedUint32(array, index, value);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ StoreIndexedUint32(array, temp, value);
+ }
}
break;
}
case kTypedDataFloat32ArrayCid:
if (IsExternal()) {
- Unsupported(compiler);
- UNREACHABLE();
- }
- if (index_scale() == 1) {
- __ StoreIndexedFloat32(array, index, value);
- } else if (index_scale() == 4) {
- __ StoreIndexed4Float32(array, index, value);
+ if (index_scale() == 1) {
+ __ StoreIndexedUntaggedFloat32(array, index, value);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ StoreIndexedUntaggedFloat32(array, temp, value);
+ }
} else {
- __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
- __ StoreIndexedFloat32(array, temp, value);
+ if (index_scale() == 1) {
+ __ StoreIndexedFloat32(array, index, value);
+ } else if (index_scale() == 4) {
+ __ StoreIndexed4Float32(array, index, value);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ StoreIndexedFloat32(array, temp, value);
+ }
}
break;
case kTypedDataFloat64ArrayCid:
if (IsExternal()) {
- Unsupported(compiler);
- UNREACHABLE();
- }
- if (index_scale() == 1) {
- __ StoreIndexedFloat64(array, index, value);
- } else if (index_scale() == 8) {
- __ StoreIndexed8Float64(array, index, value);
+ if (index_scale() == 1) {
+ __ StoreIndexedUntaggedFloat64(array, index, value);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ StoreIndexedUntaggedFloat64(array, temp, value);
+ }
} else {
- __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
- __ StoreIndexedFloat64(array, temp, value);
+ if (index_scale() == 1) {
+ __ StoreIndexedFloat64(array, index, value);
+ } else if (index_scale() == 8) {
+ __ StoreIndexed8Float64(array, index, value);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ StoreIndexedFloat64(array, temp, value);
+ }
}
break;
default:
@@ -829,7 +844,7 @@
case kExternalTypedDataUint8ClampedArrayCid:
ASSERT(index_scale() == 1);
if (IsExternal()) {
- __ LoadIndexedExternalUint8(result, array, index);
+ __ LoadIndexedUntaggedUint8(result, array, index);
} else {
__ LoadIndexedUint8(result, array, index);
}
@@ -837,7 +852,7 @@
case kTypedDataInt8ArrayCid:
ASSERT(index_scale() == 1);
if (IsExternal()) {
- __ LoadIndexedExternalInt8(result, array, index);
+ __ LoadIndexedUntaggedInt8(result, array, index);
} else {
__ LoadIndexedInt8(result, array, index);
}
@@ -861,55 +876,75 @@
case kTypedDataInt32ArrayCid:
ASSERT(representation() == kUnboxedInt32);
if (IsExternal()) {
- Unsupported(compiler);
- UNREACHABLE();
- }
- if (index_scale() == 1) {
- __ LoadIndexedInt32(result, array, index);
+ if (index_scale() == 1) {
+ __ LoadIndexedUntaggedInt32(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedUntaggedInt32(result, array, temp);
+ }
} else {
- __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
- __ LoadIndexedInt32(result, array, temp);
+ if (index_scale() == 1) {
+ __ LoadIndexedInt32(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedInt32(result, array, temp);
+ }
}
break;
case kTypedDataUint32ArrayCid:
ASSERT(representation() == kUnboxedUint32);
if (IsExternal()) {
- Unsupported(compiler);
- UNREACHABLE();
- }
- if (index_scale() == 1) {
- __ LoadIndexedUint32(result, array, index);
+ if (index_scale() == 1) {
+ __ LoadIndexedUntaggedUint32(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedUntaggedUint32(result, array, temp);
+ }
} else {
- __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
- __ LoadIndexedUint32(result, array, temp);
+ if (index_scale() == 1) {
+ __ LoadIndexedUint32(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedUint32(result, array, temp);
+ }
}
break;
case kTypedDataFloat32ArrayCid:
if (IsExternal()) {
- Unsupported(compiler);
- UNREACHABLE();
- }
- if (index_scale() == 1) {
- __ LoadIndexedFloat32(result, array, index);
- } else if (index_scale() == 4) {
- __ LoadIndexed4Float32(result, array, index);
+ if (index_scale() == 1) {
+ __ LoadIndexedUntaggedFloat32(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedUntaggedFloat32(result, array, temp);
+ }
} else {
- __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
- __ LoadIndexedFloat32(result, array, temp);
+ if (index_scale() == 1) {
+ __ LoadIndexedFloat32(result, array, index);
+ } else if (index_scale() == 4) {
+ __ LoadIndexed4Float32(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedFloat32(result, array, temp);
+ }
}
break;
case kTypedDataFloat64ArrayCid:
if (IsExternal()) {
- Unsupported(compiler);
- UNREACHABLE();
- }
- if (index_scale() == 1) {
- __ LoadIndexedFloat64(result, array, index);
- } else if (index_scale() == 8) {
- __ LoadIndexed8Float64(result, array, index);
+ if (index_scale() == 1) {
+ __ LoadIndexedUntaggedFloat64(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedUntaggedFloat64(result, array, temp);
+ }
} else {
- __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
- __ LoadIndexedFloat64(result, array, temp);
+ if (index_scale() == 1) {
+ __ LoadIndexedFloat64(result, array, index);
+ } else if (index_scale() == 8) {
+ __ LoadIndexed8Float64(result, array, index);
+ } else {
+ __ ShlImm(temp, index, Utils::ShiftForPowerOfTwo(index_scale()));
+ __ LoadIndexedFloat64(result, array, temp);
+ }
}
break;
default:
diff --git a/runtime/vm/constants_dbc.h b/runtime/vm/constants_dbc.h
index 74e1ac1..7d27ec6 100644
--- a/runtime/vm/constants_dbc.h
+++ b/runtime/vm/constants_dbc.h
@@ -866,14 +866,17 @@
V(AllocateTOpt, A_D, reg, lit, ___) \
V(StoreIndexedTOS, 0, ___, ___, ___) \
V(StoreIndexed, A_B_C, reg, reg, reg) \
- V(StoreIndexedUint8, A_B_C, reg, reg, reg) \
- V(StoreIndexedExternalUint8, A_B_C, reg, reg, reg) \
V(StoreIndexedOneByteString, A_B_C, reg, reg, reg) \
+ V(StoreIndexedUint8, A_B_C, reg, reg, reg) \
V(StoreIndexedUint32, A_B_C, reg, reg, reg) \
V(StoreIndexedFloat32, A_B_C, reg, reg, reg) \
V(StoreIndexed4Float32, A_B_C, reg, reg, reg) \
V(StoreIndexedFloat64, A_B_C, reg, reg, reg) \
V(StoreIndexed8Float64, A_B_C, reg, reg, reg) \
+ V(StoreIndexedUntaggedUint8, A_B_C, reg, reg, reg) \
+ V(StoreIndexedUntaggedUint32, A_B_C, reg, reg, reg) \
+ V(StoreIndexedUntaggedFloat32, A_B_C, reg, reg, reg) \
+ V(StoreIndexedUntaggedFloat64, A_B_C, reg, reg, reg) \
V(NoSuchMethod, 0, ___, ___, ___) \
V(TailCall, 0, ___, ___, ___) \
V(TailCallOpt, A_D, reg, reg, ___) \
@@ -885,18 +888,22 @@
V(StoreFpRelativeSlotOpt, A_B_Y, reg, reg, reg) \
V(LoadIndexedTOS, 0, ___, ___, ___) \
V(LoadIndexed, A_B_C, reg, reg, reg) \
+ V(LoadIndexedOneByteString, A_B_C, reg, reg, reg) \
+ V(LoadIndexedTwoByteString, A_B_C, reg, reg, reg) \
V(LoadIndexedUint8, A_B_C, reg, reg, reg) \
V(LoadIndexedInt8, A_B_C, reg, reg, reg) \
V(LoadIndexedInt32, A_B_C, reg, reg, reg) \
V(LoadIndexedUint32, A_B_C, reg, reg, reg) \
- V(LoadIndexedExternalUint8, A_B_C, reg, reg, reg) \
- V(LoadIndexedExternalInt8, A_B_C, reg, reg, reg) \
V(LoadIndexedFloat32, A_B_C, reg, reg, reg) \
V(LoadIndexed4Float32, A_B_C, reg, reg, reg) \
V(LoadIndexedFloat64, A_B_C, reg, reg, reg) \
V(LoadIndexed8Float64, A_B_C, reg, reg, reg) \
- V(LoadIndexedOneByteString, A_B_C, reg, reg, reg) \
- V(LoadIndexedTwoByteString, A_B_C, reg, reg, reg) \
+ V(LoadIndexedUntaggedInt8, A_B_C, reg, reg, reg) \
+ V(LoadIndexedUntaggedUint8, A_B_C, reg, reg, reg) \
+ V(LoadIndexedUntaggedInt32, A_B_C, reg, reg, reg) \
+ V(LoadIndexedUntaggedUint32, A_B_C, reg, reg, reg) \
+ V(LoadIndexedUntaggedFloat32, A_B_C, reg, reg, reg) \
+ V(LoadIndexedUntaggedFloat64, A_B_C, reg, reg, reg) \
V(StoreField, A_B_C, reg, num, reg) \
V(StoreFieldExt, A_D, reg, reg, ___) \
V(StoreFieldTOS, D, num, ___, ___) \
diff --git a/runtime/vm/simulator_dbc.cc b/runtime/vm/simulator_dbc.cc
index 85e98a0..1714769 100644
--- a/runtime/vm/simulator_dbc.cc
+++ b/runtime/vm/simulator_dbc.cc
@@ -2353,9 +2353,8 @@
{
BYTECODE(StoreIndexedFloat32, A_B_C);
uint8_t* data = SimulatorHelpers::GetTypedData(FP[rA], FP[rB]);
- const uint64_t value = reinterpret_cast<uint64_t>(FP[rC]);
- const uint32_t value32 = value;
- *reinterpret_cast<uint32_t*>(data) = value32;
+ const float value = *reinterpret_cast<float*>(&FP[rC]);
+ *reinterpret_cast<float*>(data) = value;
DISPATCH();
}
@@ -2365,10 +2364,8 @@
RawTypedData* array = reinterpret_cast<RawTypedData*>(FP[rA]);
RawSmi* index = RAW_CAST(Smi, FP[rB]);
ASSERT(SimulatorHelpers::CheckIndex(index, array->ptr()->length_));
- const uint64_t value = reinterpret_cast<uint64_t>(FP[rC]);
- const uint32_t value32 = value;
- reinterpret_cast<uint32_t*>(array->ptr()->data())[Smi::Value(index)] =
- value32;
+ const float value = *reinterpret_cast<float*>(&FP[rC]);
+ reinterpret_cast<float*>(array->ptr()->data())[Smi::Value(index)] = value;
DISPATCH();
}
@@ -3699,7 +3696,7 @@
}
{
- BYTECODE(StoreIndexedExternalUint8, A_B_C);
+ BYTECODE(StoreIndexedUntaggedUint8, A_B_C);
uint8_t* array = reinterpret_cast<uint8_t*>(FP[rA]);
RawSmi* index = RAW_CAST(Smi, FP[rB]);
RawSmi* value = RAW_CAST(Smi, FP[rC]);
@@ -3708,6 +3705,33 @@
}
{
+ BYTECODE(StoreIndexedUntaggedUint32, A_B_C);
+ uint8_t* array = reinterpret_cast<uint8_t*>(FP[rA]);
+ RawSmi* index = RAW_CAST(Smi, FP[rB]);
+ const uint32_t value = *reinterpret_cast<uint32_t*>(&FP[rC]);
+ *reinterpret_cast<uint32_t*>(array + Smi::Value(index)) = value;
+ DISPATCH();
+ }
+
+ {
+ BYTECODE(StoreIndexedUntaggedFloat32, A_B_C);
+ uint8_t* array = reinterpret_cast<uint8_t*>(FP[rA]);
+ RawSmi* index = RAW_CAST(Smi, FP[rB]);
+ const float value = *reinterpret_cast<float*>(&FP[rC]);
+ *reinterpret_cast<float*>(array + Smi::Value(index)) = value;
+ DISPATCH();
+ }
+
+ {
+ BYTECODE(StoreIndexedUntaggedFloat64, A_B_C);
+ uint8_t* array = reinterpret_cast<uint8_t*>(FP[rA]);
+ RawSmi* index = RAW_CAST(Smi, FP[rB]);
+ const double value = *reinterpret_cast<double*>(&FP[rC]);
+ *reinterpret_cast<double*>(array + Smi::Value(index)) = value;
+ DISPATCH();
+ }
+
+ {
BYTECODE(StoreIndexedOneByteString, A_B_C);
RawOneByteString* array = RAW_CAST(OneByteString, FP[rA]);
RawSmi* index = RAW_CAST(Smi, FP[rB]);
@@ -3837,7 +3861,7 @@
BYTECODE(LoadIndexedUint32, A_B_C);
const uint8_t* data = SimulatorHelpers::GetTypedData(FP[rB], FP[rC]);
const uint32_t value = *reinterpret_cast<const uint32_t*>(data);
- FP[rA] = reinterpret_cast<RawObject*>(value);
+ *reinterpret_cast<uint32_t*>(&FP[rA]) = value;
DISPATCH();
}
@@ -3845,23 +3869,60 @@
BYTECODE(LoadIndexedInt32, A_B_C);
const uint8_t* data = SimulatorHelpers::GetTypedData(FP[rB], FP[rC]);
const int32_t value = *reinterpret_cast<const int32_t*>(data);
- FP[rA] = reinterpret_cast<RawObject*>(value);
+ *reinterpret_cast<int32_t*>(&FP[rA]) = value;
DISPATCH();
}
{
- BYTECODE(LoadIndexedExternalUint8, A_B_C);
+ BYTECODE(LoadIndexedUntaggedInt8, A_B_C);
uint8_t* data = reinterpret_cast<uint8_t*>(FP[rB]);
RawSmi* index = RAW_CAST(Smi, FP[rC]);
- FP[rA] = Smi::New(data[Smi::Value(index)]);
+ FP[rA] = Smi::New(*reinterpret_cast<int8_t*>(data + Smi::Value(index)));
DISPATCH();
}
{
- BYTECODE(LoadIndexedExternalInt8, A_B_C);
- int8_t* data = reinterpret_cast<int8_t*>(FP[rB]);
+ BYTECODE(LoadIndexedUntaggedUint8, A_B_C);
+ uint8_t* data = reinterpret_cast<uint8_t*>(FP[rB]);
RawSmi* index = RAW_CAST(Smi, FP[rC]);
- FP[rA] = Smi::New(data[Smi::Value(index)]);
+ FP[rA] = Smi::New(*reinterpret_cast<uint8_t*>(data + Smi::Value(index)));
+ DISPATCH();
+ }
+
+ {
+ BYTECODE(LoadIndexedUntaggedInt32, A_B_C);
+ uint8_t* data = reinterpret_cast<uint8_t*>(FP[rB]);
+ RawSmi* index = RAW_CAST(Smi, FP[rC]);
+ const int32_t value = *reinterpret_cast<int32_t*>(data + Smi::Value(index));
+ *reinterpret_cast<int32_t*>(&FP[rA]) = value;
+ DISPATCH();
+ }
+
+ {
+ BYTECODE(LoadIndexedUntaggedUint32, A_B_C);
+ uint8_t* data = reinterpret_cast<uint8_t*>(FP[rB]);
+ RawSmi* index = RAW_CAST(Smi, FP[rC]);
+ const uint32_t value =
+ *reinterpret_cast<uint32_t*>(data + Smi::Value(index));
+ *reinterpret_cast<uint32_t*>(&FP[rA]) = value;
+ DISPATCH();
+ }
+
+ {
+ BYTECODE(LoadIndexedUntaggedFloat32, A_B_C);
+ uint8_t* data = reinterpret_cast<uint8_t*>(FP[rB]);
+ RawSmi* index = RAW_CAST(Smi, FP[rC]);
+ const float value = *reinterpret_cast<float*>(data + Smi::Value(index));
+ *reinterpret_cast<float*>(&FP[rA]) = value;
+ DISPATCH();
+ }
+
+ {
+ BYTECODE(LoadIndexedUntaggedFloat64, A_B_C);
+ uint8_t* data = reinterpret_cast<uint8_t*>(FP[rB]);
+ RawSmi* index = RAW_CAST(Smi, FP[rC]);
+ const double value = *reinterpret_cast<double*>(data + Smi::Value(index));
+ *reinterpret_cast<double*>(&FP[rA]) = value;
DISPATCH();
}
