[vm/compiler] Bug fix in 64-bit MOD (%) operator.

Rationale:
While writing tests for the ongoing native 64-bit
MOD/TRUNCDIV support in AOT, I noticed a floating-point
crash in our VM due to evaluating the constant mod case:

  Expect.equals(0, mod(minInt64, -1));
  Expect.equals(minInt64, truncdiv(minInt64, -1));

This fixes the constant evaluation part.

https://github.com/dart-lang/sdk/issues/33967

Change-Id: I9a4e6b3cd4d0d0dee39c690d2b981b5812501be4
Reviewed-on: https://dart-review.googlesource.com/67281
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Aart Bik <ajcbik@google.com>
diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc
index 057eff5..ea85486 100644
--- a/runtime/vm/object.cc
+++ b/runtime/vm/object.cc
@@ -19418,11 +19418,15 @@
         // Division special case: overflow in int64_t.
         // MIN_VALUE / -1 = (MAX_VALUE + 1), which wraps around to MIN_VALUE
         return Integer::New(Mint::kMinValue, space);
-      } else {
-        return Integer::New(left_value / right_value, space);
       }
+      return Integer::New(left_value / right_value, space);
 
     case Token::kMOD: {
+      if ((left_value == Mint::kMinValue) && (right_value == -1)) {
+        // Modulo special case: overflow in int64_t.
+        // MIN_VALUE % -1 = 0 for reason given above.
+        return Integer::New(0, space);
+      }
       const int64_t remainder = left_value % right_value;
       if (remainder < 0) {
         if (right_value < 0) {
diff --git a/tests/language_2/vm/modtruncdiv_int_test.dart b/tests/language_2/vm/modtruncdiv_int_test.dart
new file mode 100644
index 0000000..9d3f222
--- /dev/null
+++ b/tests/language_2/vm/modtruncdiv_int_test.dart
@@ -0,0 +1,214 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// VMOptions=--no_background_compilation --optimization_counter_threshold=10
+
+import "package:expect/expect.dart";
+
+// Tests for long trunc div and mod under
+// 64-bit arithmetic wrap-around semantics.
+
+final int maxInt32 = 2147483647;
+final int minInt32 = -2147483648;
+final int maxInt64 = 0x7fffffffffffffff;
+final int minInt64 = 0x8000000000000000;
+
+int mod(int x, int y) {
+  return x % y;
+}
+
+int truncdiv(int x, int y) {
+  return x ~/ y;
+}
+
+doModConstants() {
+  Expect.equals(0, mod(0, 1));
+  Expect.equals(0, mod(0, -1));
+  Expect.equals(1, mod(1, 2));
+  Expect.equals(1, mod(-1, 2));
+  Expect.equals(1, mod(1, -2));
+  Expect.equals(1, mod(-1, -2));
+  Expect.equals(2, mod(8, 3));
+  Expect.equals(1, mod(-8, 3));
+  Expect.equals(2, mod(8, -3));
+  Expect.equals(1, mod(-8, -3));
+  Expect.equals(0, mod(6, 3));
+  Expect.equals(1, mod(7, 3));
+  Expect.equals(2, mod(8, 3));
+  Expect.equals(0, mod(9, 3));
+
+  Expect.equals(1, mod(1, maxInt32));
+  Expect.equals(1, mod(1, maxInt64));
+  Expect.equals(1, mod(1, minInt32));
+  Expect.equals(1, mod(1, minInt64));
+
+  Expect.equals(maxInt32 - 1, mod(-1, maxInt32));
+  Expect.equals(maxInt64 - 1, mod(-1, maxInt64));
+  Expect.equals(maxInt32, mod(-1, minInt32));
+  Expect.equals(maxInt64, mod(-1, minInt64));
+
+  Expect.equals(0, mod(minInt32, -1));
+  Expect.equals(0, mod(maxInt32, -1));
+  Expect.equals(0, mod(minInt64, -1));
+  Expect.equals(0, mod(maxInt64, -1));
+
+  Expect.equals(0, mod(maxInt32, maxInt32));
+  Expect.equals(maxInt32, mod(maxInt32, minInt32));
+  Expect.equals(maxInt32, mod(maxInt32, maxInt64));
+  Expect.equals(maxInt32, mod(maxInt32, minInt64));
+
+  Expect.equals(maxInt32 - 1, mod(minInt32, maxInt32));
+  Expect.equals(0, mod(minInt32, minInt32));
+  Expect.equals(9223372034707292159, mod(minInt32, maxInt64));
+  Expect.equals(9223372034707292160, mod(minInt32, minInt64));
+
+  Expect.equals(1, mod(maxInt64, maxInt32));
+  Expect.equals(0, mod(maxInt64 - 1, maxInt32));
+  Expect.equals(maxInt32 - 1, mod(maxInt64 - 2, maxInt32));
+  Expect.equals(maxInt32, mod(maxInt64, minInt32));
+  Expect.equals(0, mod(maxInt64, maxInt64));
+  Expect.equals(maxInt64, mod(maxInt64, minInt64));
+
+  Expect.equals(maxInt32 - 2, mod(minInt64, maxInt32));
+  Expect.equals(0, mod(minInt64, minInt32));
+  Expect.equals(maxInt64 - 1, mod(minInt64, maxInt64));
+  Expect.equals(0, mod(minInt64, minInt64));
+
+  Expect.equals(maxInt32 - 1, mod(maxInt32 - 1, maxInt32));
+  Expect.equals(1, mod(maxInt32 + 1, maxInt32));
+  Expect.equals(maxInt32 - 2, mod(minInt32 - 1, maxInt32));
+  Expect.equals(0, mod(minInt32 + 1, maxInt32));
+}
+
+doTruncDivConstants() {
+  Expect.equals(0, truncdiv(0, 1));
+  Expect.equals(0, truncdiv(0, -1));
+  Expect.equals(0, truncdiv(1, 2));
+  Expect.equals(0, truncdiv(-1, 2));
+  Expect.equals(0, truncdiv(1, -2));
+  Expect.equals(0, truncdiv(-1, -2));
+  Expect.equals(2, truncdiv(8, 3));
+  Expect.equals(-2, truncdiv(-8, 3));
+  Expect.equals(-2, truncdiv(8, -3));
+  Expect.equals(2, truncdiv(-8, -3));
+  Expect.equals(2, truncdiv(6, 3));
+  Expect.equals(2, truncdiv(7, 3));
+  Expect.equals(2, truncdiv(8, 3));
+  Expect.equals(3, truncdiv(9, 3));
+
+  Expect.equals(0, truncdiv(1, maxInt32));
+  Expect.equals(0, truncdiv(1, maxInt64));
+  Expect.equals(0, truncdiv(1, minInt32));
+  Expect.equals(0, truncdiv(1, minInt64));
+
+  Expect.equals(0, truncdiv(-1, maxInt32));
+  Expect.equals(0, truncdiv(-1, maxInt64));
+  Expect.equals(0, truncdiv(-1, minInt32));
+  Expect.equals(0, truncdiv(-1, minInt64));
+
+  Expect.equals(-minInt32, truncdiv(minInt32, -1));
+  Expect.equals(-maxInt32, truncdiv(maxInt32, -1));
+  Expect.equals(minInt64, truncdiv(minInt64, -1));
+  Expect.equals(-maxInt64, truncdiv(maxInt64, -1));
+
+  Expect.equals(1, truncdiv(maxInt32, maxInt32));
+  Expect.equals(0, truncdiv(maxInt32, minInt32));
+  Expect.equals(0, truncdiv(maxInt32, maxInt64));
+  Expect.equals(0, truncdiv(maxInt32, minInt64));
+
+  Expect.equals(-1, truncdiv(minInt32, maxInt32));
+  Expect.equals(1, truncdiv(minInt32, minInt32));
+  Expect.equals(0, truncdiv(minInt32, maxInt64));
+  Expect.equals(0, truncdiv(minInt32, minInt64));
+
+  Expect.equals(4294967298, truncdiv(maxInt64, maxInt32));
+  Expect.equals(4294967298, truncdiv(maxInt64 - 1, maxInt32));
+  Expect.equals(4294967297, truncdiv(maxInt64 - 2, maxInt32));
+  Expect.equals(-4294967295, truncdiv(maxInt64, minInt32));
+  Expect.equals(1, truncdiv(maxInt64, maxInt64));
+  Expect.equals(0, truncdiv(maxInt64, minInt64));
+
+  Expect.equals(-4294967298, truncdiv(minInt64, maxInt32));
+  Expect.equals(4294967296, truncdiv(minInt64, minInt32));
+  Expect.equals(-1, truncdiv(minInt64, maxInt64));
+  Expect.equals(1, truncdiv(minInt64, minInt64));
+
+  Expect.equals(0, truncdiv(maxInt32 - 1, maxInt32));
+  Expect.equals(1, truncdiv(maxInt32 + 1, maxInt32));
+  Expect.equals(-1, truncdiv(minInt32 - 1, maxInt32));
+  Expect.equals(-1, truncdiv(minInt32 + 1, maxInt32));
+}
+
+int acc;
+
+doModVars(int xlo, int xhi, int ylo, int yhi) {
+  for (int x = xlo; x <= xhi; x++) {
+    for (int y = ylo; y <= yhi; y++) {
+      acc += mod(x, y);
+    }
+  }
+}
+
+doTruncDivVars(int xlo, int xhi, int ylo, int yhi) {
+  for (int x = xlo; x <= xhi; x++) {
+    for (int y = ylo; y <= yhi; y++) {
+      acc += truncdiv(x, y);
+    }
+  }
+}
+
+main() {
+  // Repeat constant tests to enter JIT (when applicable).
+
+  for (int i = 0; i < 20; i++) {
+    doModConstants();
+    doTruncDivConstants();
+  }
+
+  // Variable ranges.
+
+  acc = 0;
+  doModVars(3, 5, 2, 6);
+  Expect.equals(28, acc);
+
+  acc = 0;
+  doModVars((3 << 32) - 1, (3 << 32) + 1, (3 << 32) - 1, (3 << 32) + 1);
+  Expect.equals(38654705666, acc);
+
+  acc = 0;
+  doTruncDivVars(3, 5, 2, 6);
+  Expect.equals(11, acc);
+
+  acc = 0;
+  doTruncDivVars(-5, -3, 2, 6);
+  Expect.equals(-11, acc);
+
+  acc = 0;
+  doTruncDivVars(3, 5, -6, -2);
+  Expect.equals(-11, acc);
+
+  acc = 0;
+  doTruncDivVars(-5, -3, -6, -2);
+  Expect.equals(11, acc);
+
+  acc = 0;
+  doTruncDivVars((3 << 32) - 1, (3 << 32) + 1, 3, 6);
+  Expect.equals(36721970376, acc);
+
+  // Exceptions at the right time.
+
+  acc = 0;
+  try {
+    doModVars(9, 9, -9, 0);
+    acc = 0; // don't reach!
+  } catch (e, s) {}
+  Expect.equals(12, acc);
+
+  acc = 0;
+  try {
+    doTruncDivVars(9, 9, -9, 0);
+    acc = 0; // don't reach!
+  } catch (e, s) {}
+  Expect.equals(-23, acc);
+}