[vm/compiler] Various 64-bit operator improvements.
Rationale:
Improves the slow path of 64-bit REM/TRUNCDIV on
X64 and ARM64. Also introduces 64-bit negate operator,
which was needed to expose negative contants
(viz. x / -3 was represented as x / - (3) first).
The negate operator is not recognized in AOT mode yet,
since shifts by out-of-range constants should learn
how to throw rather than deopt....
https://github.com/dart-lang/sdk/issues/33967
https://github.com/flutter/flutter/issues/19677
Change-Id: I7d81c9b1c72d99e8c4018f68c0501c7b599e073f
Reviewed-on: https://dart-review.googlesource.com/68280
Commit-Queue: Aart Bik <ajcbik@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
diff --git a/runtime/vm/compiler/aot/aot_call_specializer.cc b/runtime/vm/compiler/aot/aot_call_specializer.cc
index 77c7bf5..11fe560 100644
--- a/runtime/vm/compiler/aot/aot_call_specializer.cc
+++ b/runtime/vm/compiler/aot/aot_call_specializer.cc
@@ -565,7 +565,6 @@
}
break;
}
- // TODO(dartbug.com/30480): Enable 64-bit integer shifts (SHL, SHR).
case Token::kBIT_OR:
case Token::kBIT_XOR:
case Token::kBIT_AND:
@@ -608,7 +607,6 @@
}
break;
}
-
case Token::kSHL:
case Token::kSHR: {
Value* left_value = instr->PushArgumentAt(receiver_index)->value();
@@ -622,7 +620,6 @@
}
break;
}
-
default:
break;
}
diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h
index 50507fc..ff8df89 100644
--- a/runtime/vm/compiler/backend/il.h
+++ b/runtime/vm/compiler/backend/il.h
@@ -6076,9 +6076,13 @@
class UnaryInt64OpInstr : public UnaryIntegerOpInstr {
public:
- UnaryInt64OpInstr(Token::Kind op_kind, Value* value, intptr_t deopt_id)
- : UnaryIntegerOpInstr(op_kind, value, deopt_id) {
- ASSERT(op_kind == Token::kBIT_NOT);
+ UnaryInt64OpInstr(Token::Kind op_kind,
+ Value* value,
+ intptr_t deopt_id,
+ SpeculativeMode speculative_mode = kGuardInputs)
+ : UnaryIntegerOpInstr(op_kind, value, deopt_id),
+ speculative_mode_(speculative_mode) {
+ ASSERT(op_kind == Token::kBIT_NOT || op_kind == Token::kNEGATE);
}
virtual bool ComputeCanDeoptimize() const { return false; }
@@ -6092,9 +6096,17 @@
return kUnboxedInt64;
}
+ virtual bool AttributesEqual(Instruction* other) const {
+ return UnaryIntegerOpInstr::AttributesEqual(other) &&
+ (speculative_mode() == other->speculative_mode());
+ }
+
+ virtual SpeculativeMode speculative_mode() const { return speculative_mode_; }
+
DECLARE_INSTRUCTION(UnaryInt64Op)
private:
+ const SpeculativeMode speculative_mode_;
DISALLOW_COPY_AND_ASSIGN(UnaryInt64OpInstr);
};
diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc
index f1c8664..0b8f0e3 100644
--- a/runtime/vm/compiler/backend/il_arm64.cc
+++ b/runtime/vm/compiler/backend/il_arm64.cc
@@ -5047,7 +5047,8 @@
static const intptr_t kNumberOfArguments = 0;
Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
- Register right,
+ Register divisor,
+ Range* divisor_range,
Register tmp,
Register out,
intptr_t try_index)
@@ -5056,36 +5057,63 @@
kNumberOfArguments,
try_index),
is_mod_(instruction->op_kind() == Token::kMOD),
- right_(right),
+ divisor_(divisor),
+ divisor_range_(divisor_range),
tmp_(tmp),
out_(out),
adjust_sign_label_() {}
void EmitNativeCode(FlowGraphCompiler* compiler) override {
- // Main entry throws, use code of superclass.
- ThrowErrorSlowPathCode::EmitNativeCode(compiler);
- // Adjust modulo for negative sign.
- // if (right < 0)
- // out -= right;
+ // Handle modulo/division by zero, if needed. Use superclass code.
+ if (has_divide_by_zero()) {
+ ThrowErrorSlowPathCode::EmitNativeCode(compiler);
+ } else {
+ __ Bind(entry_label()); // not used, but keeps destructor happy
+ if (Assembler::EmittingComments()) {
+ __ Comment("slow path %s operation (no throw)", name());
+ }
+ }
+ // Adjust modulo for negative sign, optimized for known ranges.
+ // if (divisor < 0)
+ // out -= divisor;
// else
- // out += right;
- if (is_mod_) {
+ // out += divisor;
+ if (has_adjust_sign()) {
__ Bind(adjust_sign_label());
- __ CompareRegisters(right_, ZR);
- __ sub(tmp_, out_, Operand(right_));
- __ add(out_, out_, Operand(right_));
- __ csel(out_, tmp_, out_, LT);
+ if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
+ // General case.
+ __ CompareRegisters(divisor_, ZR);
+ __ sub(tmp_, out_, Operand(divisor_));
+ __ add(out_, out_, Operand(divisor_));
+ __ csel(out_, tmp_, out_, LT);
+ } else if (divisor_range_->IsPositive()) {
+ // Always positive.
+ __ add(out_, out_, Operand(divisor_));
+ } else {
+ // Always negative.
+ __ sub(out_, out_, Operand(divisor_));
+ }
__ b(exit_label());
}
}
const char* name() override { return "int64 divide"; }
- Label* adjust_sign_label() { return &adjust_sign_label_; }
+ bool has_divide_by_zero() { return RangeUtils::CanBeZero(divisor_range_); }
+
+ bool has_adjust_sign() { return is_mod_; }
+
+ bool is_needed() { return has_divide_by_zero() || has_adjust_sign(); }
+
+ Label* adjust_sign_label() {
+ ASSERT(has_adjust_sign());
+ return &adjust_sign_label_;
+ }
private:
bool is_mod_;
- Register right_;
+ Register divisor_;
+ Range* divisor_range_;
Register tmp_;
Register out_;
Label adjust_sign_label_;
@@ -5100,14 +5128,16 @@
Register out) {
ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);
- // Set up a slow path.
+ // Prepare a slow path.
+ Range* right_range = instruction->right()->definition()->range();
Int64DivideSlowPath* slow_path = new (Z) Int64DivideSlowPath(
- instruction, right, tmp, out, compiler->CurrentTryIndex());
- compiler->AddSlowPathCode(slow_path);
+ instruction, right, right_range, tmp, out, compiler->CurrentTryIndex());
// Handle modulo/division by zero exception on slow path.
- __ CompareRegisters(right, ZR);
- __ b(slow_path->entry_label(), EQ);
+ if (slow_path->has_divide_by_zero()) {
+ __ CompareRegisters(right, ZR);
+ __ b(slow_path->entry_label(), EQ);
+ }
// Perform actual operation
// out = left % right
@@ -5124,7 +5154,10 @@
__ sdiv(out, left, right);
}
- __ Bind(slow_path->exit_label());
+ if (slow_path->is_needed()) {
+ __ Bind(slow_path->exit_label());
+ compiler->AddSlowPathCode(slow_path);
+ }
}
LocationSummary* BinaryInt64OpInstr::MakeLocationSummary(Zone* zone,
@@ -5571,11 +5604,18 @@
}
void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
- ASSERT(op_kind() == Token::kBIT_NOT);
const Register left = locs()->in(0).reg();
const Register out = locs()->out(0).reg();
- ASSERT(out == left);
- __ mvn(out, left);
+ switch (op_kind()) {
+ case Token::kBIT_NOT:
+ __ mvn(out, left);
+ break;
+ case Token::kNEGATE:
+ __ sub(out, ZR, Operand(left));
+ break;
+ default:
+ UNREACHABLE();
+ }
}
CompileType BinaryUint32OpInstr::ComputeType() const {
diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc
index 5ef9a71..f36a8b2 100644
--- a/runtime/vm/compiler/backend/il_x64.cc
+++ b/runtime/vm/compiler/backend/il_x64.cc
@@ -5201,54 +5201,96 @@
static const intptr_t kNumberOfArguments = 0;
Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
- Register right,
+ Register divisor,
+ Range* divisor_range,
intptr_t try_index)
: ThrowErrorSlowPathCode(instruction,
kIntegerDivisionByZeroExceptionRuntimeEntry,
kNumberOfArguments,
try_index),
is_mod_(instruction->op_kind() == Token::kMOD),
- right_(right),
+ divisor_(divisor),
+ divisor_range_(divisor_range),
div_by_minus_one_label_(),
adjust_sign_label_() {}
void EmitNativeCode(FlowGraphCompiler* compiler) override {
- // Main entry throws, use code of superclass.
- ThrowErrorSlowPathCode::EmitNativeCode(compiler);
- // Handle modulo/division by minus one.
- __ Bind(div_by_minus_one_label());
- if (is_mod_) {
- __ xorq(RDX, RDX); // x % -1 = 0
+ // Handle modulo/division by zero, if needed. Use superclass code.
+ if (has_divide_by_zero()) {
+ ThrowErrorSlowPathCode::EmitNativeCode(compiler);
} else {
- __ negq(RAX); // x / -1 = -x
+ __ Bind(entry_label()); // not used, but keeps destructor happy
+ if (Assembler::EmittingComments()) {
+ __ Comment("slow path %s operation (no throw)", name());
+ }
}
- __ jmp(exit_label());
- // Adjust modulo for negative sign.
- // if (right < 0)
- // out -= right;
- // else
- // out += right;
- if (is_mod_) {
- Label subtract;
- __ Bind(adjust_sign_label());
- __ testq(right_, right_);
- __ j(LESS, &subtract, Assembler::kNearJump);
- __ addq(RDX, right_);
+ // Handle modulo/division by minus one, if needed.
+ // Note: an exact -1 divisor is best optimized prior to codegen.
+ if (has_divide_by_minus_one()) {
+ __ Bind(div_by_minus_one_label());
+ if (is_mod_) {
+ __ xorq(RDX, RDX); // x % -1 = 0
+ } else {
+ __ negq(RAX); // x / -1 = -x
+ }
__ jmp(exit_label());
- __ Bind(&subtract);
- __ subq(RDX, right_);
+ }
+ // Adjust modulo for negative sign, optimized for known ranges.
+ // if (divisor < 0)
+ // out -= divisor;
+ // else
+ // out += divisor;
+ if (has_adjust_sign()) {
+ __ Bind(adjust_sign_label());
+ if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
+ // General case.
+ Label subtract;
+ __ testq(divisor_, divisor_);
+ __ j(LESS, &subtract, Assembler::kNearJump);
+ __ addq(RDX, divisor_);
+ __ jmp(exit_label());
+ __ Bind(&subtract);
+ __ subq(RDX, divisor_);
+ } else if (divisor_range_->IsPositive()) {
+ // Always positive.
+ __ addq(RDX, divisor_);
+ } else {
+ // Always negative.
+ __ subq(RDX, divisor_);
+ }
__ jmp(exit_label());
}
}
const char* name() override { return "int64 divide"; }
- Label* div_by_minus_one_label() { return &div_by_minus_one_label_; }
- Label* adjust_sign_label() { return &adjust_sign_label_; }
+ bool has_divide_by_zero() { return RangeUtils::CanBeZero(divisor_range_); }
+
+ bool has_divide_by_minus_one() {
+ return RangeUtils::Overlaps(divisor_range_, -1, -1);
+ }
+
+ bool has_adjust_sign() { return is_mod_; }
+
+ bool is_needed() {
+ return has_divide_by_zero() || has_divide_by_minus_one() ||
+ has_adjust_sign();
+ }
+
+ Label* div_by_minus_one_label() {
+ ASSERT(has_divide_by_minus_one());
+ return &div_by_minus_one_label_;
+ }
+
+ Label* adjust_sign_label() {
+ ASSERT(has_adjust_sign());
+ return &adjust_sign_label_;
+ }
private:
bool is_mod_;
- Register right_;
+ Register divisor_;
+ Range* divisor_range_;
Label div_by_minus_one_label_;
Label adjust_sign_label_;
};
@@ -5262,19 +5304,23 @@
Register out) {
ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);
- // Set up a slow path.
- Int64DivideSlowPath* slow_path = new (Z)
- Int64DivideSlowPath(instruction, right, compiler->CurrentTryIndex());
- compiler->AddSlowPathCode(slow_path);
+ // Prepare a slow path.
+ Range* right_range = instruction->right()->definition()->range();
+ Int64DivideSlowPath* slow_path = new (Z) Int64DivideSlowPath(
+ instruction, right, right_range, compiler->CurrentTryIndex());
// Handle modulo/division by zero exception on slow path.
- __ testq(right, right);
- __ j(EQUAL, slow_path->entry_label());
+ if (slow_path->has_divide_by_zero()) {
+ __ testq(right, right);
+ __ j(EQUAL, slow_path->entry_label());
+ }
// Handle modulo/division by minus one explicitly on slow path
// (to avoid arithmetic exception on 0x8000000000000000 / -1).
- __ cmpq(right, Immediate(-1));
- __ j(EQUAL, slow_path->div_by_minus_one_label());
+ if (slow_path->has_divide_by_minus_one()) {
+ __ cmpq(right, Immediate(-1));
+ __ j(EQUAL, slow_path->div_by_minus_one_label());
+ }
// Perform actual operation
// out = left % right
@@ -5294,7 +5340,11 @@
ASSERT(out == RAX);
ASSERT(tmp == RDX);
}
- __ Bind(slow_path->exit_label());
+
+ if (slow_path->is_needed()) {
+ __ Bind(slow_path->exit_label());
+ compiler->AddSlowPathCode(slow_path);
+ }
}
template <typename OperandType>
@@ -5393,11 +5443,19 @@
}
void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
- ASSERT(op_kind() == Token::kBIT_NOT);
const Register left = locs()->in(0).reg();
const Register out = locs()->out(0).reg();
ASSERT(out == left);
- __ notq(left);
+ switch (op_kind()) {
+ case Token::kBIT_NOT:
+ __ notq(left);
+ break;
+ case Token::kNEGATE:
+ __ negq(left);
+ break;
+ default:
+ UNREACHABLE();
+ }
}
static void EmitShiftInt64ByConstant(FlowGraphCompiler* compiler,