commit cb08bf287de1b987a627ce3a4c3495310a844d86
author Alexander Thomas <athom@google.com> Wed Apr 04 11:51:25 2018 +0000
committer Alexander Thomas <athom@google.com> Wed Apr 04 11:51:25 2018 +0000
tree f7287c175aab63a05ca4a7a97e64a8112ca3756f
parent b793c8a3870dfee553dbf9691e3b5bb404d7ec84

[infra] Expect certain errors to pass when runtime is "none"

test.py ignores RuntimeError, MissingRuntimeError, and Timeout when the
runtime is "none". Previously, the status update tool would attempt to fix
these "unexpected passes".

Change-Id: I73c15fdc7e18aa29d5553e58b6b4e65ec07c1246
Reviewed-on: https://dart-review.googlesource.com/49461
Reviewed-by: William Hesse <whesse@google.com>

tools/gardening/lib/src/results/status_expectations.dart

diff --git a/tools/gardening/lib/src/results/status_expectations.dart b/tools/gardening/lib/src/results/status_expectations.dart
index 5131a48..58bd4e8 100644
--- a/tools/gardening/lib/src/results/status_expectations.dart
+++ b/tools/gardening/lib/src/results/status_expectations.dart
@@ -112,9 +112,17 @@
     Set<Expectation> testExpectations =
         expectationsFromTest(result.testExpectations);
     Set<Expectation> expectationSet = expectations();
+    final allowedWithoutRuntime = [
+      Expectation.runtimeError,
+      Expectation.missingRuntimeError,
+      Expectation.timeout
+    ];
     _isSuccess = testExpectations.contains(outcome) ||
         expectationSet.contains(Expectation.skip) ||
         expectationSet.contains(Expectation.skipByDesign) ||
+        outcome == Expectation.pass &&
+            configuration.runtime == "none" &&
+            allowedWithoutRuntime.any(expectationSet.contains) ||
         expectationSet.any((expectation) {
           return outcome.canBeOutcomeOf(expectation);
         });