[vm] Enable test pattern (a&b == 0) fusion in AOT on X64/ARM64
Previously we would only fuse this pattern for Smi operations
which (almost) never happens in AOT. This CL enables fusion
for Int64 operations as well.
The implementation is limited to X64 and ARM64 for now
because implementing it on 32-bit platforms is somewhat
cumbersome.
Issue https://github.com/dart-lang/sdk/issues/55522
R=alexmarkov@google.com
TEST=vm/cc/IL_TestIntInstr,vm/dart/test_int_pattern_il_test
Cq-Include-Trybots: luci.dart.try:vm-linux-release-simarm-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-simarm_x64-try,dart-sdk-linux-riscv64-try,vm-aot-linux-debug-simriscv64-try,vm-ffi-qemu-linux-release-riscv64-try,vm-linux-debug-simriscv64-try,vm-linux-release-ia32-try
Change-Id: I62a482640db45befac6b0b78850f23a8cc624c75
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/365463
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Slava Egorov <vegorov@google.com>
diff --git a/pkg/vm/lib/testing/il_matchers.dart b/pkg/vm/lib/testing/il_matchers.dart
index 5990b7c..f7f2051 100644
--- a/pkg/vm/lib/testing/il_matchers.dart
+++ b/pkg/vm/lib/testing/il_matchers.dart
@@ -829,3 +829,17 @@
runtimeConfiguration.endsWith('ARM_X64') ||
runtimeConfiguration.endsWith('RISCV32');
})();
+
+final String _config = (() {
+ if (bool.hasEnvironment(testRunnerKey)) {
+ return const String.fromEnvironment(testRunnerKey);
+ } else if (Platform.environment['DART_CONFIGURATION']
+ case final runtimeConfiguration?) {
+ return runtimeConfiguration.toLowerCase();
+ } else {
+ throw 'Expected either $testRunnerKey or DART_CONFIGURATION to be defined';
+ }
+})();
+
+final bool isArm64 = _config.endsWith('arm64');
+final bool isX64 = _config.endsWith('x64') && !_config.endsWith('arm_x64');
diff --git a/runtime/tests/vm/dart/test_int_pattern_il_test.dart b/runtime/tests/vm/dart/test_int_pattern_il_test.dart
new file mode 100644
index 0000000..c7aa27c
--- /dev/null
+++ b/runtime/tests/vm/dart/test_int_pattern_il_test.dart
@@ -0,0 +1,63 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// This test verifies fusing of (a & b) == 0 patterns.
+
+import 'package:expect/expect.dart';
+import 'package:vm/testing/il_matchers.dart';
+
+@pragma('vm:never-inline')
+@pragma('vm:testing:print-flow-graph')
+String testValue0(int value) => (value & 1) == 0 ? "f" : "t";
+
+@pragma('vm:never-inline')
+@pragma('vm:testing:print-flow-graph')
+String testValue1(int value) => (value & 3) == 0 ? "f" : "t";
+
+final List<String Function(int)> tests = [
+ testValue0,
+ testValue1,
+];
+
+void main() {
+ for (var j = 0; j < tests.length; j++) {
+ Expect.equals("f", tests[j](0), "mismatch at input 0 test $j");
+ Expect.equals("f", tests[j](4), "mismatch at input 4 test $j");
+ Expect.equals("t", tests[j](1), "mismatch at input 1 test $j");
+ Expect.equals("t", tests[j](3), "mismatch at input 3 test $j");
+ }
+}
+
+void matchIL$testValue0(FlowGraph graph) {
+ if (!isX64 && !isArm64) {
+ return;
+ }
+ graph.match([
+ match.block('Graph', [
+ 'int64(1)' << match.UnboxedConstant(value: 1),
+ ]),
+ match.block('Function', [
+ 'value' << match.Parameter(index: 0),
+ 'unbox(value)' << match.UnboxInt64('value'),
+ match.Branch(match.TestInt('unbox(value)', 'int64(1)')),
+ ])
+ ]);
+}
+
+void matchIL$testValue1(FlowGraph graph) {
+ if (!isX64 && !isArm64) {
+ return;
+ }
+
+ graph.match([
+ match.block('Graph', [
+ 'int64(3)' << match.UnboxedConstant(value: 3),
+ ]),
+ match.block('Function', [
+ 'value' << match.Parameter(index: 0),
+ 'unbox(value)' << match.UnboxInt64('value'),
+ match.Branch(match.TestInt('unbox(value)', 'int64(3)')),
+ ])
+ ]);
+}
diff --git a/runtime/vm/compiler/backend/block_builder.h b/runtime/vm/compiler/backend/block_builder.h
index 41c6424..d04eb4b 100644
--- a/runtime/vm/compiler/backend/block_builder.h
+++ b/runtime/vm/compiler/backend/block_builder.h
@@ -64,6 +64,10 @@
DartReturnInstr* AddReturn(Value* value) {
const auto& function = flow_graph_->function();
const auto representation = FlowGraph::ReturnRepresentationOf(function);
+ return AddReturn(value, representation);
+ }
+
+ DartReturnInstr* AddReturn(Value* value, Representation representation) {
DartReturnInstr* instr = new DartReturnInstr(
Source(), value, CompilerState::Current().GetNextDeoptId(),
representation);
diff --git a/runtime/vm/compiler/backend/constant_propagator.cc b/runtime/vm/compiler/backend/constant_propagator.cc
index 6c1cd74..405cbee 100644
--- a/runtime/vm/compiler/backend/constant_propagator.cc
+++ b/runtime/vm/compiler/backend/constant_propagator.cc
@@ -670,7 +670,7 @@
// Comparison instruction that is equivalent to the (left & right) == 0
// comparison pattern.
-void ConstantPropagator::VisitTestSmi(TestSmiInstr* instr) {
+void ConstantPropagator::VisitTestInt(TestIntInstr* instr) {
const Object& left = instr->left()->definition()->constant_value();
const Object& right = instr->right()->definition()->constant_value();
if (IsNonConstant(left) || IsNonConstant(right)) {
diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc
index 903e26e..ff80294 100644
--- a/runtime/vm/compiler/backend/il.cc
+++ b/runtime/vm/compiler/backend/il.cc
@@ -3618,7 +3618,7 @@
return false;
}
- BinarySmiOpInstr* mask_op = left->definition()->AsBinarySmiOp();
+ auto mask_op = left->definition()->AsBinaryIntegerOp();
if ((mask_op == nullptr) || (mask_op->op_kind() != Token::kBIT_AND) ||
!mask_op->HasOnlyUse(left)) {
return false;
@@ -3686,30 +3686,37 @@
}
if (comparison()->IsEqualityCompare() &&
- comparison()->operation_cid() == kSmiCid) {
- BinarySmiOpInstr* bit_and = nullptr;
- bool negate = false;
- if (RecognizeTestPattern(comparison()->left(), comparison()->right(),
- &negate)) {
- bit_and = comparison()->left()->definition()->AsBinarySmiOp();
- } else if (RecognizeTestPattern(comparison()->right(), comparison()->left(),
- &negate)) {
- bit_and = comparison()->right()->definition()->AsBinarySmiOp();
- }
- if (bit_and != nullptr) {
- if (FLAG_trace_optimization && flow_graph->should_print()) {
- THR_Print("Merging test smi v%" Pd "\n", bit_and->ssa_temp_index());
+ (comparison()->operation_cid() == kSmiCid ||
+ comparison()->operation_cid() == kMintCid)) {
+ const auto representation =
+ comparison()->operation_cid() == kSmiCid ? kTagged : kUnboxedInt64;
+ if (TestIntInstr::IsSupported(representation)) {
+ BinaryIntegerOpInstr* bit_and = nullptr;
+ bool negate = false;
+ if (RecognizeTestPattern(comparison()->left(), comparison()->right(),
+ &negate)) {
+ bit_and = comparison()->left()->definition()->AsBinaryIntegerOp();
+ } else if (RecognizeTestPattern(comparison()->right(),
+ comparison()->left(), &negate)) {
+ bit_and = comparison()->right()->definition()->AsBinaryIntegerOp();
}
- TestSmiInstr* test = new TestSmiInstr(
- comparison()->source(),
- negate ? Token::NegateComparison(comparison()->kind())
- : comparison()->kind(),
- bit_and->left()->Copy(zone), bit_and->right()->Copy(zone));
- ASSERT(!CanDeoptimize());
- RemoveEnvironment();
- flow_graph->CopyDeoptTarget(this, bit_and);
- SetComparison(test);
- bit_and->RemoveFromGraph();
+ if (bit_and != nullptr) {
+ if (FLAG_trace_optimization && flow_graph->should_print()) {
+ THR_Print("Merging test integer v%" Pd "\n",
+ bit_and->ssa_temp_index());
+ }
+ TestIntInstr* test = new TestIntInstr(
+ comparison()->source(),
+ negate ? Token::NegateComparison(comparison()->kind())
+ : comparison()->kind(),
+ representation, bit_and->left()->Copy(zone),
+ bit_and->right()->Copy(zone));
+ ASSERT(!CanDeoptimize());
+ RemoveEnvironment();
+ flow_graph->CopyDeoptTarget(this, bit_and);
+ SetComparison(test);
+ bit_and->RemoveFromGraph();
+ }
}
}
return this;
@@ -6569,9 +6576,10 @@
needs_number_check(), DeoptId::kNone);
}
-ComparisonInstr* TestSmiInstr::CopyWithNewOperands(Value* new_left,
+ComparisonInstr* TestIntInstr::CopyWithNewOperands(Value* new_left,
Value* new_right) {
- return new TestSmiInstr(source(), kind(), new_left, new_right);
+ return new TestIntInstr(source(), kind(), representation_, new_left,
+ new_right);
}
ComparisonInstr* TestCidsInstr::CopyWithNewOperands(Value* new_left,
@@ -8568,6 +8576,31 @@
// No-op.
}
+int64_t TestIntInstr::ComputeImmediateMask() {
+ int64_t mask = Integer::Cast(locs()->in(1).constant()).AsInt64Value();
+
+ switch (representation_) {
+ case kTagged:
+ // If operand is tagged we need to tag the mask.
+ if (!Smi::IsValid(mask)) {
+ // Mask it not a valid Smi. This means top bits are not all equal to
+ // the sign bit and at least some of them are 1. If they were all
+ // 0 than it would be a valid positive Smi.
+ // Adjust the mask to make it a valid Smi: testing any bit above
+ // kSmiBits is equivalent to testing the sign bit.
+ mask = (mask & kSmiMax) | kSmiMin;
+ }
+ return compiler::target::ToRawSmi(mask);
+
+ case kUnboxedInt64:
+ return mask;
+
+ default:
+ UNREACHABLE();
+ return -1;
+ }
+}
+
#undef __
} // namespace dart
diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h
index 39b1c99..0f78136 100644
--- a/runtime/vm/compiler/backend/il.h
+++ b/runtime/vm/compiler/backend/il.h
@@ -526,7 +526,7 @@
M(GuardFieldType, _) \
M(IfThenElse, kNoGC) \
M(MaterializeObject, _) \
- M(TestSmi, kNoGC) \
+ M(TestInt, kNoGC) \
M(TestCids, kNoGC) \
M(TestRange, kNoGC) \
M(ExtractNthOutput, kNoGC) \
@@ -5137,19 +5137,21 @@
// Comparison instruction that is equivalent to the (left & right) == 0
// comparison pattern.
-class TestSmiInstr : public TemplateComparison<2, NoThrow, Pure> {
+class TestIntInstr : public TemplateComparison<2, NoThrow, Pure> {
public:
- TestSmiInstr(const InstructionSource& source,
+ TestIntInstr(const InstructionSource& source,
Token::Kind kind,
+ Representation representation,
Value* left,
Value* right)
- : TemplateComparison(source, kind) {
+ : TemplateComparison(source, kind), representation_(representation) {
ASSERT(kind == Token::kEQ || kind == Token::kNE);
+ ASSERT(IsSupported(representation));
SetInputAt(0, left);
SetInputAt(1, right);
}
- DECLARE_COMPARISON_INSTRUCTION(TestSmi);
+ DECLARE_COMPARISON_INSTRUCTION(TestInt);
virtual ComparisonInstr* CopyWithNewOperands(Value* left, Value* right);
@@ -5158,13 +5160,42 @@
virtual bool ComputeCanDeoptimize() const { return false; }
virtual Representation RequiredInputRepresentation(intptr_t idx) const {
- return kTagged;
+ return representation_;
}
- DECLARE_EMPTY_SERIALIZATION(TestSmiInstr, TemplateComparison)
+ virtual SpeculativeMode SpeculativeModeOfInput(intptr_t index) const {
+ return kNotSpeculative;
+ }
+
+ static bool IsSupported(Representation representation) {
+ switch (representation) {
+ case kTagged:
+#if defined(TARGET_ARCH_X64) || defined(TARGET_ARCH_ARM64) || \
+ defined(TARGET_ARCH_RISCV64)
+ case kUnboxedInt64:
+#endif
+ return true;
+
+ default:
+ return false;
+ }
+ }
+
+#if defined(TARGET_ARCH_ARM64)
+ virtual void EmitBranchCode(FlowGraphCompiler* compiler, BranchInstr* branch);
+#endif
+
+#define FIELD_LIST(F) F(const Representation, representation_)
+
+ DECLARE_INSTRUCTION_SERIALIZABLE_FIELDS(TestIntInstr,
+ TemplateComparison,
+ FIELD_LIST)
+#undef FIELD_LIST
private:
- DISALLOW_COPY_AND_ASSIGN(TestSmiInstr);
+ int64_t ComputeImmediateMask();
+
+ DISALLOW_COPY_AND_ASSIGN(TestIntInstr);
};
// Checks the input value cid against cids stored in a table and returns either
diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc
index c8bd8ae..f612746 100644
--- a/runtime/vm/compiler/backend/il_arm.cc
+++ b/runtime/vm/compiler/backend/il_arm.cc
@@ -1553,7 +1553,8 @@
}
}
-LocationSummary* TestSmiInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+LocationSummary* TestIntInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+ RELEASE_ASSERT(representation_ == kTagged);
const intptr_t kNumInputs = 2;
const intptr_t kNumTemps = 0;
LocationSummary* locs = new (zone)
@@ -1562,17 +1563,16 @@
// Only one input can be a constant operand. The case of two constant
// operands should be handled by constant propagation.
locs->set_in(1, LocationRegisterOrConstant(right()));
+ locs->set_out(0, Location::RequiresRegister());
return locs;
}
-Condition TestSmiInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
+Condition TestIntInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
BranchLabels labels) {
const Register left = locs()->in(0).reg();
Location right = locs()->in(1);
if (right.IsConstant()) {
- ASSERT(compiler::target::IsSmi(right.constant()));
- const int32_t imm = compiler::target::ToRawSmi(right.constant());
- __ TestImmediate(left, imm);
+ __ TestImmediate(left, static_cast<int32_t>(ComputeImmediateMask()));
} else {
__ tst(left, compiler::Operand(right.reg()));
}
diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc
index 1688b08..a956ba4 100644
--- a/runtime/vm/compiler/backend/il_arm64.cc
+++ b/runtime/vm/compiler/backend/il_arm64.cc
@@ -1296,7 +1296,7 @@
}
}
-LocationSummary* TestSmiInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+LocationSummary* TestIntInstr::MakeLocationSummary(Zone* zone, bool opt) const {
const intptr_t kNumInputs = 2;
const intptr_t kNumTemps = 0;
LocationSummary* locs = new (zone)
@@ -1305,24 +1305,82 @@
// Only one input can be a constant operand. The case of two constant
// operands should be handled by constant propagation.
locs->set_in(1, LocationRegisterOrConstant(right()));
+ locs->set_out(0, Location::RequiresRegister());
return locs;
}
-Condition TestSmiInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
+Condition TestIntInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
BranchLabels labels) {
const Register left = locs()->in(0).reg();
Location right = locs()->in(1);
+ const auto operand_size = representation_ == kTagged ? compiler::kObjectBytes
+ : compiler::kEightBytes;
if (right.IsConstant()) {
- ASSERT(right.constant().IsSmi());
- const int64_t imm = Smi::RawValue(Smi::Cast(right.constant()).Value());
- __ TestImmediate(left, imm, compiler::kObjectBytes);
+ __ TestImmediate(left, ComputeImmediateMask(), operand_size);
} else {
- __ tst(left, compiler::Operand(right.reg()), compiler::kObjectBytes);
+ __ tst(left, compiler::Operand(right.reg()), operand_size);
}
Condition true_condition = (kind() == Token::kNE) ? NE : EQ;
return true_condition;
}
+static bool IsSingleBitMask(Location mask, intptr_t* bit) {
+ if (!mask.IsConstant()) {
+ return false;
+ }
+
+ uint64_t mask_value =
+ static_cast<uint64_t>(Integer::Cast(mask.constant()).AsInt64Value());
+ if (!Utils::IsPowerOfTwo(mask_value)) {
+ return false;
+ }
+
+ *bit = Utils::CountTrailingZeros64(mask_value);
+ return true;
+}
+
+void TestIntInstr::EmitBranchCode(FlowGraphCompiler* compiler,
+ BranchInstr* branch) {
+ // Check if this is a single bit test. In this case this branch can be
+ // emitted as TBZ/TBNZ.
+ intptr_t bit_index;
+ if (IsSingleBitMask(locs()->in(1), &bit_index)) {
+ BranchLabels labels = compiler->CreateBranchLabels(branch);
+ const Register value = locs()->in(0).reg();
+
+ bool branch_on_zero_bit;
+ bool can_fallthrough;
+ compiler::Label* target;
+ if (labels.fall_through == labels.true_label) {
+ target = labels.false_label;
+ branch_on_zero_bit = (kind() == Token::kNE);
+ can_fallthrough = true;
+ } else {
+ target = labels.true_label;
+ branch_on_zero_bit = (kind() == Token::kEQ);
+ can_fallthrough = (labels.fall_through == labels.false_label);
+ }
+
+ if (representation_ == kTagged) {
+ bit_index = Utils::Minimum(kSmiBits, bit_index) + kSmiTagShift;
+ }
+
+ if (branch_on_zero_bit) {
+ __ tbz(target, value, bit_index);
+ } else {
+ __ tbnz(target, value, bit_index);
+ }
+ if (!can_fallthrough) {
+ __ b(labels.false_label);
+ }
+
+ return;
+ }
+
+ // Otherwise use shared implementation.
+ ComparisonInstr::EmitBranchCode(compiler, branch);
+}
+
LocationSummary* TestCidsInstr::MakeLocationSummary(Zone* zone,
bool opt) const {
const intptr_t kNumInputs = 1;
diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc
index 475c738..932c6f1 100644
--- a/runtime/vm/compiler/backend/il_ia32.cc
+++ b/runtime/vm/compiler/backend/il_ia32.cc
@@ -1032,7 +1032,8 @@
}
}
-LocationSummary* TestSmiInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+LocationSummary* TestIntInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+ RELEASE_ASSERT(representation_ == kTagged);
const intptr_t kNumInputs = 2;
const intptr_t kNumTemps = 0;
LocationSummary* locs = new (zone)
@@ -1041,17 +1042,17 @@
// Only one input can be a constant operand. The case of two constant
// operands should be handled by constant propagation.
locs->set_in(1, LocationRegisterOrConstant(right()));
+ locs->set_out(0, Location::RequiresRegister());
return locs;
}
-Condition TestSmiInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
+Condition TestIntInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
BranchLabels labels) {
Register left = locs()->in(0).reg();
Location right = locs()->in(1);
if (right.IsConstant()) {
- ASSERT(right.constant().IsSmi());
- const int32_t imm = static_cast<int32_t>(right.constant().ptr());
- __ testl(left, compiler::Immediate(imm));
+ __ testl(left,
+ compiler::Immediate(static_cast<int32_t>(ComputeImmediateMask())));
} else {
__ testl(left, right.reg());
}
diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc
index 0aff43c..e393384 100644
--- a/runtime/vm/compiler/backend/il_riscv.cc
+++ b/runtime/vm/compiler/backend/il_riscv.cc
@@ -1412,7 +1412,7 @@
}
}
-LocationSummary* TestSmiInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+LocationSummary* TestIntInstr::MakeLocationSummary(Zone* zone, bool opt) const {
const intptr_t kNumInputs = 2;
const intptr_t kNumTemps = 0;
LocationSummary* locs = new (zone)
@@ -1421,17 +1421,16 @@
// Only one input can be a constant operand. The case of two constant
// operands should be handled by constant propagation.
locs->set_in(1, LocationRegisterOrConstant(right()));
+ locs->set_out(0, Location::RequiresRegister());
return locs;
}
-Condition TestSmiInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
+Condition TestIntInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
BranchLabels labels) {
const Register left = locs()->in(0).reg();
Location right = locs()->in(1);
if (right.IsConstant()) {
- ASSERT(right.constant().IsSmi());
- const intx_t imm = static_cast<intx_t>(right.constant().ptr());
- __ TestImmediate(left, imm);
+ __ TestImmediate(left, ComputeImmediateMask());
} else {
__ TestRegisters(left, right.reg());
}
diff --git a/runtime/vm/compiler/backend/il_test.cc b/runtime/vm/compiler/backend/il_test.cc
index 07dfe58..c6eff83 100644
--- a/runtime/vm/compiler/backend/il_test.cc
+++ b/runtime/vm/compiler/backend/il_test.cc
@@ -4,12 +4,15 @@
#include "vm/compiler/backend/il.h"
+#include <optional>
#include <vector>
#include "platform/text_buffer.h"
#include "platform/utils.h"
#include "vm/class_id.h"
+#include "vm/compiler/assembler/disassembler.h"
#include "vm/compiler/backend/block_builder.h"
+#include "vm/compiler/backend/flow_graph_compiler.h"
#include "vm/compiler/backend/il_printer.h"
#include "vm/compiler/backend/il_test_helper.h"
#include "vm/compiler/backend/range_analysis.h"
@@ -1529,4 +1532,200 @@
/*expected_to_forward=*/false);
}
+template <typename... Args>
+static ObjectPtr InvokeFunction(const Function& function, Args&... args) {
+ const Array& args_array = Array::Handle(Array::New(sizeof...(Args)));
+ intptr_t i = 0;
+ (args_array.SetAt(i++, args), ...);
+ return DartEntry::InvokeFunction(function, args_array);
+}
+
+static const Function& BuildTestFunction(
+ intptr_t num_parameters,
+ std::function<void(FlowGraphBuilderHelper&)> build_graph) {
+ using compiler::BlockBuilder;
+
+ TestPipeline pipeline(CompilerPass::kAOT, [&]() {
+ FlowGraphBuilderHelper H(num_parameters);
+ build_graph(H);
+ H.FinishGraph();
+ return H.flow_graph();
+ });
+ auto flow_graph = pipeline.RunPasses({
+ CompilerPass::kFinalizeGraph,
+ CompilerPass::kReorderBlocks,
+ CompilerPass::kAllocateRegisters,
+ });
+ pipeline.CompileGraphAndAttachFunction();
+ return flow_graph->function();
+}
+
+enum class TestIntVariant {
+ kTestBranch,
+ kTestValue,
+};
+
+static const Function& BuildTestIntFunction(
+ Zone* zone,
+ TestIntVariant test_variant,
+ bool eq_zero,
+ Representation rep,
+ std::optional<int64_t> immediate_mask) {
+ using compiler::BlockBuilder;
+ return BuildTestFunction(
+ /*num_parameters=*/1 + (!immediate_mask.has_value() ? 1 : 0),
+ [&](auto& H) {
+ H.AddVariable("lhs", AbstractType::ZoneHandle(Type::IntType()),
+ new CompileType(CompileType::Int()));
+ if (!immediate_mask.has_value()) {
+ H.AddVariable("rhs", AbstractType::ZoneHandle(Type::IntType()),
+ new CompileType(CompileType::Int()));
+ }
+
+ auto normal_entry = H.flow_graph()->graph_entry()->normal_entry();
+ auto true_successor = H.TargetEntry();
+ auto false_successor = H.TargetEntry();
+
+ {
+ BlockBuilder builder(H.flow_graph(), normal_entry);
+ Definition* lhs = builder.AddParameter(0);
+ Definition* rhs = immediate_mask.has_value()
+ ? H.IntConstant(immediate_mask.value(), rep)
+ : builder.AddParameter(1);
+ if (rep != lhs->representation()) {
+ lhs =
+ builder.AddUnboxInstr(kUnboxedInt64, lhs, /*is_checked=*/false);
+ }
+ if (rep != rhs->representation()) {
+ rhs =
+ builder.AddUnboxInstr(kUnboxedInt64, rhs, /*is_checked=*/false);
+ }
+
+ auto comparison = new TestIntInstr(
+ InstructionSource(), eq_zero ? Token::kEQ : Token::kNE, rep,
+ new Value(lhs), new Value(rhs));
+
+ if (test_variant == TestIntVariant::kTestValue) {
+ auto v2 = builder.AddDefinition(comparison);
+ builder.AddReturn(new Value(v2));
+ } else {
+ builder.AddBranch(comparison, true_successor, false_successor);
+ }
+ }
+
+ if (test_variant == TestIntVariant::kTestBranch) {
+ {
+ BlockBuilder builder(H.flow_graph(), true_successor);
+ builder.AddReturn(
+ new Value(H.flow_graph()->GetConstant(Bool::True())));
+ }
+
+ {
+ BlockBuilder builder(H.flow_graph(), false_successor);
+ builder.AddReturn(
+ new Value(H.flow_graph()->GetConstant(Bool::False())));
+ }
+ }
+ });
+}
+
+static void TestIntTestWithImmediate(Zone* zone,
+ TestIntVariant test_variant,
+ bool eq_zero,
+ Representation rep,
+ const std::vector<int64_t>& inputs,
+ int64_t mask) {
+ const auto& func =
+ BuildTestIntFunction(zone, test_variant, eq_zero, rep, mask);
+ auto invoke = [&](int64_t v) -> bool {
+ const auto& input = Integer::Handle(Integer::New(v));
+ EXPECT(rep == kUnboxedInt64 || input.IsSmi());
+ const auto& result = Bool::CheckedHandle(zone, InvokeFunction(func, input));
+ return result.value();
+ };
+
+ for (auto& input : inputs) {
+ const auto expected = ((input & mask) == 0) == eq_zero;
+ const auto got = invoke(input);
+ if (expected != got) {
+ FAIL("testing [%s] [%s] %" Px64 " & %" Px64
+ " %s 0: expected %s but got %s\n",
+ test_variant == TestIntVariant::kTestBranch ? "branch" : "value",
+ RepresentationUtils::ToCString(rep), input, mask,
+ eq_zero ? "==" : "!=", expected ? "true" : "false",
+ got ? "true" : "false");
+ }
+ }
+}
+
+static void TestIntTest(Zone* zone,
+ TestIntVariant test_variant,
+ bool eq_zero,
+ Representation rep,
+ const std::vector<int64_t>& inputs,
+ const std::vector<int64_t>& masks) {
+ if (!TestIntInstr::IsSupported(rep)) {
+ return;
+ }
+
+ const auto& func = BuildTestIntFunction(zone, test_variant, eq_zero, rep, {});
+ auto invoke = [&](int64_t lhs, int64_t mask) -> bool {
+ const auto& arg0 = Integer::Handle(Integer::New(lhs));
+ const auto& arg1 = Integer::Handle(Integer::New(mask));
+ EXPECT(rep == kUnboxedInt64 || arg0.IsSmi());
+ EXPECT(rep == kUnboxedInt64 || arg1.IsSmi());
+ const auto& result =
+ Bool::CheckedHandle(zone, InvokeFunction(func, arg0, arg1));
+ return result.value();
+ };
+
+ for (auto& mask : masks) {
+ TestIntTestWithImmediate(zone, test_variant, eq_zero, rep, inputs, mask);
+
+ // We allow non-Smi masks as immediates but not as non-constant operands.
+ if (rep == kTagged && !Smi::IsValid(mask)) {
+ continue;
+ }
+
+ for (auto& input : inputs) {
+ const auto expected = ((input & mask) == 0) == eq_zero;
+ const auto got = invoke(input, mask);
+ if (expected != got) {
+ FAIL("testing [%s] [%s] %" Px64 " & %" Px64
+ " %s 0: expected %s but got %s\n",
+ test_variant == TestIntVariant::kTestBranch ? "branch" : "value",
+ RepresentationUtils::ToCString(rep), input, mask,
+ eq_zero ? "==" : "!=", expected ? "true" : "false",
+ got ? "true" : "false");
+ }
+ }
+ }
+}
+
+ISOLATE_UNIT_TEST_CASE(IL_TestIntInstr) {
+ const int64_t msb = static_cast<int64_t>(0x8000000000000000L);
+ const int64_t kSmiSignBit = kSmiMax + 1;
+
+ const std::initializer_list<int64_t> kMasks = {
+ 1, 2, kSmiSignBit, kSmiSignBit | 1, msb, msb | 1};
+
+ const std::vector<std::pair<Representation, std::vector<int64_t>>> kValues = {
+ {kTagged,
+ {-2, -1, 0, 1, 2, 3, kSmiMax & ~1, kSmiMin & ~1, kSmiMax | 1,
+ kSmiMin | 1}},
+ {kUnboxedInt64,
+ {-2, -1, 0, 1, 2, 3, kSmiMax & ~1, kSmiMin & ~1, kSmiMax | 1,
+ kSmiMin | 1, msb, msb | 1, msb | 2}},
+ };
+
+ for (auto test_variant :
+ {TestIntVariant::kTestBranch, TestIntVariant::kTestValue}) {
+ for (auto eq_zero : {true, false}) {
+ for (auto& [rep, values] : kValues) {
+ TestIntTest(thread->zone(), test_variant, eq_zero, rep, values, kMasks);
+ }
+ }
+ }
+}
+
} // namespace dart
diff --git a/runtime/vm/compiler/backend/il_test_helper.cc b/runtime/vm/compiler/backend/il_test_helper.cc
index 0dbdb03..162b700 100644
--- a/runtime/vm/compiler/backend/il_test_helper.cc
+++ b/runtime/vm/compiler/backend/il_test_helper.cc
@@ -120,22 +120,24 @@
const bool optimized = true;
const intptr_t osr_id = Compiler::kNoOSRDeoptId;
- auto pipeline = CompilationPipeline::New(zone, function_);
+ if (flow_graph_ == nullptr) {
+ auto pipeline = CompilationPipeline::New(zone, function_);
- parsed_function_ = new (zone)
- ParsedFunction(thread, Function::ZoneHandle(zone, function_.ptr()));
- pipeline->ParseFunction(parsed_function_);
+ parsed_function_ = new (zone)
+ ParsedFunction(thread, Function::ZoneHandle(zone, function_.ptr()));
+ pipeline->ParseFunction(parsed_function_);
- // Extract type feedback before the graph is built, as the graph
- // builder uses it to attach it to nodes.
- ic_data_array_ = new (zone) ZoneGrowableArray<const ICData*>();
- if (mode_ == CompilerPass::kJIT) {
- function_.RestoreICDataMap(ic_data_array_, /*clone_ic_data=*/false);
+ // Extract type feedback before the graph is built, as the graph
+ // builder uses it to attach it to nodes.
+ ic_data_array_ = new (zone) ZoneGrowableArray<const ICData*>();
+ if (mode_ == CompilerPass::kJIT) {
+ function_.RestoreICDataMap(ic_data_array_, /*clone_ic_data=*/false);
+ }
+
+ flow_graph_ = pipeline->BuildFlowGraph(zone, parsed_function_,
+ ic_data_array_, osr_id, optimized);
}
- flow_graph_ = pipeline->BuildFlowGraph(zone, parsed_function_, ic_data_array_,
- osr_id, optimized);
-
if (mode_ == CompilerPass::kAOT) {
flow_graph_->PopulateWithICData(function_);
}
diff --git a/runtime/vm/compiler/backend/il_test_helper.h b/runtime/vm/compiler/backend/il_test_helper.h
index b507236..0c48f4c 100644
--- a/runtime/vm/compiler/backend/il_test_helper.h
+++ b/runtime/vm/compiler/backend/il_test_helper.h
@@ -72,11 +72,10 @@
class TestPipeline : public ValueObject {
public:
- explicit TestPipeline(const Function& function,
- CompilerPass::PipelineMode mode,
- bool is_optimizing = true)
- : function_(function),
- thread_(Thread::Current()),
+ TestPipeline(const Function& function,
+ CompilerPass::PipelineMode mode,
+ bool is_optimizing = true)
+ : thread_(Thread::Current()),
compiler_state_(thread_,
mode == CompilerPass::PipelineMode::kAOT,
is_optimizing,
@@ -84,9 +83,27 @@
hierarchy_info_(thread_),
speculative_policy_(std::unique_ptr<SpeculativeInliningPolicy>(
new SpeculativeInliningPolicy(/*enable_suppresson=*/false))),
- mode_(mode) {}
+ mode_(mode),
+ flow_graph_(nullptr),
+ function_(function),
+ parsed_function_(nullptr) {}
~TestPipeline() { delete pass_state_; }
+ TestPipeline(CompilerPass::PipelineMode mode, std::function<FlowGraph*()> fn)
+ : thread_(Thread::Current()),
+ compiler_state_(thread_,
+ mode == CompilerPass::PipelineMode::kAOT,
+ /*is_optimizing=*/true,
+ CompilerTracing::kOff),
+ hierarchy_info_(thread_),
+ speculative_policy_(std::unique_ptr<SpeculativeInliningPolicy>(
+ new SpeculativeInliningPolicy(/*enable_suppresson=*/false))),
+ mode_(mode),
+ flow_graph_(fn()),
+ function_(flow_graph_->function()),
+ parsed_function_(
+ const_cast<ParsedFunction*>(&flow_graph_->parsed_function())) {}
+
// As a side-effect this will populate
// - [ic_data_array_]
// - [parsed_function_]
@@ -101,16 +118,16 @@
void CompileGraphAndAttachFunction();
private:
- const Function& function_;
Thread* thread_;
CompilerState compiler_state_;
HierarchyInfo hierarchy_info_;
std::unique_ptr<SpeculativeInliningPolicy> speculative_policy_;
CompilerPass::PipelineMode mode_;
ZoneGrowableArray<const ICData*>* ic_data_array_ = nullptr;
- ParsedFunction* parsed_function_ = nullptr;
CompilerPassState* pass_state_ = nullptr;
- FlowGraph* flow_graph_ = nullptr;
+ FlowGraph* flow_graph_;
+ const Function& function_;
+ ParsedFunction* parsed_function_;
};
// Match opcodes used for [ILMatcher], see below.
@@ -270,11 +287,15 @@
class FlowGraphBuilderHelper {
public:
- explicit FlowGraphBuilderHelper(intptr_t num_parameters = 0)
+ explicit FlowGraphBuilderHelper(
+ intptr_t num_parameters = 0,
+ const std::function<void(const Function&)>& configure_function =
+ [](const Function&) {})
: state_(CompilerState::Current()),
flow_graph_(MakeDummyGraph(Thread::Current(),
num_parameters,
- state_.is_optimizing())) {
+ state_.is_optimizing(),
+ configure_function)) {
flow_graph_.CreateCommonConstants();
}
@@ -288,9 +309,12 @@
state_.GetNextDeoptId());
}
- ConstantInstr* IntConstant(int64_t value) const {
+ ConstantInstr* IntConstant(int64_t value,
+ Representation representation = kTagged) const {
+ ASSERT(representation == kTagged ||
+ RepresentationUtils::IsUnboxedInteger(representation));
return flow_graph_.GetConstant(
- Integer::Handle(Integer::NewCanonical(value)));
+ Integer::Handle(Integer::NewCanonical(value)), representation);
}
ConstantInstr* DoubleConstant(double value) {
@@ -377,9 +401,11 @@
FlowGraph* flow_graph() { return &flow_graph_; }
private:
- static FlowGraph& MakeDummyGraph(Thread* thread,
- intptr_t num_parameters,
- bool is_optimizing) {
+ static FlowGraph& MakeDummyGraph(
+ Thread* thread,
+ intptr_t num_parameters,
+ bool is_optimizing,
+ const std::function<void(const Function&)>& configure_function) {
const FunctionType& signature =
FunctionType::ZoneHandle(FunctionType::New());
signature.set_num_fixed_parameters(num_parameters);
@@ -393,6 +419,7 @@
/*is_native=*/true,
Class::Handle(thread->isolate_group()->object_store()->object_class()),
TokenPosition::kNoSource));
+ configure_function(func);
Zone* zone = thread->zone();
ParsedFunction* parsed_function = new (zone) ParsedFunction(thread, func);
diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc
index 1ea95e1..078b8d0 100644
--- a/runtime/vm/compiler/backend/il_x64.cc
+++ b/runtime/vm/compiler/backend/il_x64.cc
@@ -1215,7 +1215,7 @@
}
}
-LocationSummary* TestSmiInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+LocationSummary* TestIntInstr::MakeLocationSummary(Zone* zone, bool opt) const {
const intptr_t kNumInputs = 2;
const intptr_t kNumTemps = 0;
LocationSummary* locs = new (zone)
@@ -1224,20 +1224,26 @@
// Only one input can be a constant operand. The case of two constant
// operands should be handled by constant propagation.
locs->set_in(1, LocationRegisterOrConstant(right()));
+ locs->set_out(0, Location::RequiresRegister());
return locs;
}
-Condition TestSmiInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
+Condition TestIntInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
BranchLabels labels) {
Register left_reg = locs()->in(0).reg();
Location right = locs()->in(1);
if (right.IsConstant()) {
- ASSERT(right.constant().IsSmi());
- const int64_t imm = Smi::RawValue(Smi::Cast(right.constant()).Value());
- __ TestImmediate(left_reg, compiler::Immediate(imm),
- compiler::kObjectBytes);
+ const auto operand_size = representation_ == kTagged
+ ? compiler::kObjectBytes
+ : compiler::kEightBytes;
+ __ TestImmediate(left_reg, compiler::Immediate(ComputeImmediateMask()),
+ operand_size);
} else {
- __ OBJ(test)(left_reg, right.reg());
+ if (representation_ == kTagged) {
+ __ OBJ(test)(left_reg, right.reg());
+ } else {
+ __ testq(left_reg, right.reg());
+ }
}
Condition true_condition = (kind() == Token::kNE) ? NOT_ZERO : ZERO;
return true_condition;
diff --git a/runtime/vm/compiler/backend/type_propagator.cc b/runtime/vm/compiler/backend/type_propagator.cc
index 42a7fd4..7f10844 100644
--- a/runtime/vm/compiler/backend/type_propagator.cc
+++ b/runtime/vm/compiler/backend/type_propagator.cc
@@ -1409,7 +1409,7 @@
return CompileType::Bool();
}
-CompileType TestSmiInstr::ComputeType() const {
+CompileType TestIntInstr::ComputeType() const {
return CompileType::Bool();
}
