[infra] Fix expected results of multitests, drop checked-only results.
All multitests expecting errors were just being reported as expecting "fail", not as expecting compile-time
or runtime error. Cleaned up legacy checked-only expectations as a side effect.
Change-Id: I7785db0e094e3af3464700fbc71aac30b8bb4282
Reviewed-on: https://dart-review.googlesource.com/c/90684
Reviewed-by: Jonas Termansen <sortie@google.com>
Auto-Submit: William Hesse <whesse@google.com>
diff --git a/tools/testing/dart/command_output.dart b/tools/testing/dart/command_output.dart
index a392933..43caac4 100644
--- a/tools/testing/dart/command_output.dart
+++ b/tools/testing/dart/command_output.dart
@@ -472,7 +472,7 @@
}
// Handle errors / missing errors
- if (testCase.expectCompileError) {
+ if (testCase.hasCompileError) {
if (errors.isNotEmpty) {
return Expectation.pass;
}
@@ -629,7 +629,7 @@
if (hasTimedOut) return Expectation.timeout;
if (hasNonUtf8) return Expectation.nonUtf8Error;
- if (testCase.expectCompileError) {
+ if (testCase.hasCompileError) {
if (testCase.hasSyntaxError) {
// A syntax error is expected.
return hasSyntaxError
@@ -676,7 +676,7 @@
if (hasNonUtf8) return Expectation.nonUtf8Error;
// Multitests are handled specially.
- if (testCase.expectCompileError) {
+ if (testCase.hasCompileError) {
if (exitCode == _compileErrorExitCode) {
return Expectation.pass;
}
@@ -794,7 +794,7 @@
}
// Multitests are handled specially.
- if (testCase.expectCompileError) {
+ if (testCase.hasCompileError) {
// Nonzero exit code of the compiler means compilation failed
// TODO(kustermann): Do we have a special exit code in that case???
if (exitCode != 0) {
@@ -836,7 +836,7 @@
if (hasNonUtf8) return Expectation.nonUtf8Error;
// Handle errors / missing errors
- if (testCase.expectCompileError) {
+ if (testCase.hasCompileError) {
return exitCode == 0
? Expectation.missingCompileTimeError
: Expectation.pass;
@@ -897,7 +897,7 @@
}
// Multitests are handled specially.
- if (testCase.expectCompileError) {
+ if (testCase.hasCompileError) {
if (exitCode == VMCommandOutput._compileErrorExitCode ||
exitCode == kBatchModeCompileTimeErrorExit) {
return Expectation.pass;
diff --git a/tools/testing/dart/multitest.dart b/tools/testing/dart/multitest.dart
index 9247c3b..319c4b8 100644
--- a/tools/testing/dart/multitest.dart
+++ b/tools/testing/dart/multitest.dart
@@ -12,9 +12,8 @@
/// lines of the file, and all of the multitest lines containing that key, in
/// the same order as in the source file. The new test is expected to pass if
/// the error type listed is 'ok', and to fail if the error type is 'syntax
-/// error', 'compile-time error', 'runtime error', 'static type warning',
-/// 'dynamic type error', or 'checked mode compile-time error'. The type error
-/// tests fail only in checked mode. There is also a test created from only the
+/// error', 'compile-time error', 'runtime error', or 'static type warning'.
+/// There is also a test created from only the
/// untagged lines of the file, with key "none", which is expected to pass. This
/// library extracts these tests, writes them into a temporary directory, and
/// passes them to the test runner. These tests may be referred to in the status
@@ -64,13 +63,10 @@
/// eee //# 10: ok
/// fff
/// ```
-///
-/// Note that it is possible to indicate more than one acceptable outcome in
-/// the case of dynamic and static type warnings
-///
+//////
/// ```dart
/// aaa
-/// ddd //# 07: static type warning, dynamic type error
+/// ddd //# 07: static type warning
/// fff
/// ```
import "dart:async";
@@ -89,9 +85,9 @@
'compile-time error',
'runtime error',
// TODO(rnystrom): Remove these after Dart 1.0 tests are removed.
- 'static type warning',
- 'dynamic type error',
- 'checked mode compile-time error'
+ 'static type warning', // This is still a valid analyzer test
+ 'dynamic type error', // This is now a no-op
+ 'checked mode compile-time error' // This is now a no-op
].toSet();
// Note: This function is called directly by:
@@ -222,24 +218,17 @@
var hasSyntaxError = outcome.contains('syntax error');
var hasCompileError =
hasSyntaxError || outcome.contains('compile-time error');
- var isNegativeIfChecked = outcome.contains('dynamic type error');
- var hasCompileErrorIfChecked =
- outcome.contains('checked mode compile-time error');
- if (hotReload) {
- if (hasCompileError || hasCompileErrorIfChecked) {
- // Running a test that expects a compilation error with hot reloading
- // is redundant with a regular run of the test.
- continue;
- }
+ if (hotReload && hasCompileError) {
+ // Running a test that expects a compilation error with hot reloading
+ // is redundant with a regular run of the test.
+ continue;
}
doTest(multitestFilename, filePath,
hasSyntaxError: hasSyntaxError,
hasCompileError: hasCompileError,
hasRuntimeError: hasRuntimeError,
- isNegativeIfChecked: isNegativeIfChecked,
- hasCompileErrorIfChecked: hasCompileErrorIfChecked,
hasStaticWarning: hasStaticWarning,
multitestKey: key);
}
diff --git a/tools/testing/dart/test_progress.dart b/tools/testing/dart/test_progress.dart
index e56e891..29117e8 100644
--- a/tools/testing/dart/test_progress.dart
+++ b/tools/testing/dart/test_progress.dart
@@ -613,18 +613,15 @@
final ranAllCommands = test.commandOutputs.length == test.commands.length;
if (!test.lastCommandOutput.hasTimedOut) {
- if (!ranAllCommands && !test.expectCompileError) {
+ if (!ranAllCommands && !test.hasCompileError) {
output.write('Unexpected compile error.');
} else {
- if (test.expectCompileError) {
+ if (test.hasCompileError) {
output.write('Missing expected compile error.');
}
if (test.hasRuntimeError) {
output.write('Missing expected runtime error.');
}
- if (test.configuration.isChecked && test.isNegativeIfChecked) {
- output.write('Missing expected dynamic type error.');
- }
}
}
}
diff --git a/tools/testing/dart/test_runner.dart b/tools/testing/dart/test_runner.dart
index 2081403..1b1d03a 100644
--- a/tools/testing/dart/test_runner.dart
+++ b/tools/testing/dart/test_runner.dart
@@ -76,14 +76,10 @@
*/
class TestCase extends UniqueObject {
// Flags set in _expectations from the optional argument info.
- static final int IS_NEGATIVE = 1 << 0;
- static final int HAS_RUNTIME_ERROR = 1 << 1;
- static final int HAS_STATIC_WARNING = 1 << 2;
- static final int IS_NEGATIVE_IF_CHECKED = 1 << 3;
- static final int HAS_SYNTAX_ERROR = 1 << 4;
- static final int HAS_COMPILE_ERROR = 1 << 5;
- static final int HAS_COMPILE_ERROR_IF_CHECKED = 1 << 6;
- static final int EXPECT_COMPILE_ERROR = 1 << 7;
+ static final int HAS_RUNTIME_ERROR = 1 << 0;
+ static final int HAS_SYNTAX_ERROR = 1 << 1;
+ static final int HAS_COMPILE_ERROR = 1 << 2;
+ static final int HAS_STATIC_WARNING = 1 << 3;
/**
* A list of commands to execute. Most test cases have a single command.
* Dart2js tests have two commands, one to compile the source and another
@@ -102,13 +98,10 @@
TestCase(this.displayName, this.commands, this.configuration,
this.expectedOutcomes,
- {bool isNegative: false, TestInformation info}) {
+ {TestInformation info}) {
// A test case should do something.
assert(commands.isNotEmpty);
- if (isNegative || displayName.contains("negative_test")) {
- _expectations |= IS_NEGATIVE;
- }
if (info != null) {
_setExpectations(info);
hash = info.originTestPath.relativeTo(Repository.dir).toString().hashCode;
@@ -119,20 +112,11 @@
// We don't want to keep the entire (large) TestInformation structure,
// so we copy the needed bools into flags set in a single integer.
if (info.hasRuntimeError) _expectations |= HAS_RUNTIME_ERROR;
- if (info.hasStaticWarning) _expectations |= HAS_STATIC_WARNING;
- if (info.isNegativeIfChecked) _expectations |= IS_NEGATIVE_IF_CHECKED;
if (info.hasSyntaxError) _expectations |= HAS_SYNTAX_ERROR;
if (info.hasCompileError || info.hasSyntaxError) {
_expectations |= HAS_COMPILE_ERROR;
}
- if (info.hasCompileErrorIfChecked) {
- _expectations |= HAS_COMPILE_ERROR_IF_CHECKED;
- }
- if (info.hasCompileError ||
- info.hasSyntaxError ||
- (configuration.isChecked && info.hasCompileErrorIfChecked)) {
- _expectations |= EXPECT_COMPILE_ERROR;
- }
+ if (info.hasStaticWarning) _expectations |= HAS_STATIC_WARNING;
}
TestCase indexedCopy(int index) {
@@ -143,15 +127,14 @@
..hash = hash;
}
- bool get isNegative => _expectations & IS_NEGATIVE != 0;
bool get hasRuntimeError => _expectations & HAS_RUNTIME_ERROR != 0;
bool get hasStaticWarning => _expectations & HAS_STATIC_WARNING != 0;
- bool get isNegativeIfChecked => _expectations & IS_NEGATIVE_IF_CHECKED != 0;
bool get hasSyntaxError => _expectations & HAS_SYNTAX_ERROR != 0;
bool get hasCompileError => _expectations & HAS_COMPILE_ERROR != 0;
- bool get hasCompileErrorIfChecked =>
- _expectations & HAS_COMPILE_ERROR_IF_CHECKED != 0;
- bool get expectCompileError => _expectations & EXPECT_COMPILE_ERROR != 0;
+ bool get isNegative =>
+ hasCompileError ||
+ hasRuntimeError && configuration.runtime != Runtime.none ||
+ displayName.contains("negative_test");
bool get unexpectedOutput {
var outcome = this.result;
@@ -163,19 +146,27 @@
Expectation get result => lastCommandOutput.result(this);
Expectation get realResult => lastCommandOutput.realResult(this);
Expectation get realExpected {
- if (isNegative || (isNegativeIfChecked && configuration.isChecked)) {
- return Expectation.fail;
- }
if (configuration.compiler == Compiler.specParser) {
if (hasSyntaxError) {
return Expectation.syntaxError;
}
- } else if ((hasCompileError) ||
- (hasCompileErrorIfChecked && configuration.isChecked)) {
+ } else if (hasCompileError) {
+ if (hasRuntimeError && configuration.runtime != Runtime.none) {
+ return Expectation.fail;
+ }
return Expectation.compileTimeError;
}
- if (configuration.runtime != Runtime.none && hasRuntimeError) {
- return Expectation.runtimeError;
+ if (hasRuntimeError) {
+ if (configuration.runtime != Runtime.none) {
+ return Expectation.runtimeError;
+ }
+ return Expectation.pass;
+ }
+ if (displayName.contains("negative_test")) {
+ return Expectation.fail;
+ }
+ if (configuration.compiler == Compiler.dart2analyzer && hasStaticWarning) {
+ return Expectation.staticWarning;
}
return Expectation.pass;
}
diff --git a/tools/testing/dart/test_suite.dart b/tools/testing/dart/test_suite.dart
index 3afc50e..e5d7c40 100644
--- a/tools/testing/dart/test_suite.dart
+++ b/tools/testing/dart/test_suite.dart
@@ -47,8 +47,6 @@
{bool hasSyntaxError,
bool hasCompileError,
bool hasRuntimeError,
- bool isNegativeIfChecked,
- bool hasCompileErrorIfChecked,
bool hasStaticWarning,
String multitestKey});
@@ -287,8 +285,7 @@
var negative = info != null ? isNegative(info) : false;
var testCase = new TestCase(
displayName, commands, configuration, expectations,
- isNegative: negative, info: info);
-
+ info: info);
if (negative &&
configuration.runtimeConfiguration.shouldSkipNegativeTests) {
return;
@@ -314,8 +311,7 @@
if (configuration.hotReload || configuration.hotReloadRollback) {
// Handle reload special cases.
if (expectations.contains(Expectation.compileTimeError) ||
- testCase.hasCompileError ||
- testCase.expectCompileError) {
+ testCase.hasCompileError) {
// Running a test that expects a compilation error with hot reloading
// is redundant with a regular run of the test.
return;
@@ -345,22 +341,9 @@
doTest(testCase);
}
- bool expectCompileError(TestInformation info) {
- return info.hasCompileError ||
- (configuration.isChecked && info.hasCompileErrorIfChecked);
- }
-
- bool isNegative(TestInformation info) {
- if (info.hasRuntimeError && configuration.runtime != Runtime.none) {
- return true;
- }
-
- if (info.isNegativeIfChecked && configuration.isChecked) {
- return true;
- }
-
- return expectCompileError(info);
- }
+ bool isNegative(TestInformation info) =>
+ info.hasCompileError ||
+ info.hasRuntimeError && configuration.runtime != Runtime.none;
String createGeneratedTestDirectoryHelper(
String name, String dirname, Path testPath) {
@@ -531,8 +514,6 @@
bool hasSyntaxError;
bool hasCompileError;
bool hasRuntimeError;
- bool isNegativeIfChecked;
- bool hasCompileErrorIfChecked;
bool hasStaticWarning;
String multitestKey;
@@ -543,8 +524,6 @@
this.hasSyntaxError,
this.hasCompileError,
this.hasRuntimeError,
- this.isNegativeIfChecked,
- this.hasCompileErrorIfChecked,
this.hasStaticWarning,
{this.multitestKey: ''}) {
assert(filePath.isAbsolute);
@@ -784,7 +763,7 @@
}
}
if (configuration.compilerConfiguration.hasCompiler &&
- expectCompileError(info)) {
+ info.hasCompileError) {
// If a compile-time error is expected, and we're testing a
// compiler, we never need to attempt to run the program (in a
// browser or otherwise).
@@ -875,7 +854,7 @@
commands.addAll(compilationArtifact.commands);
}
- if (expectCompileError(info) &&
+ if (info.hasCompileError &&
compilerConfiguration.hasCompiler &&
!compilerConfiguration.runRuntimeDespiteMissingCompileTimeError) {
// Do not attempt to run the compiled result. A compilation
@@ -908,21 +887,11 @@
{bool hasSyntaxError,
bool hasCompileError,
bool hasRuntimeError,
- bool isNegativeIfChecked: false,
- bool hasCompileErrorIfChecked: false,
bool hasStaticWarning: false,
String multitestKey}) {
// Cache the test information for each test case.
- var info = new TestInformation(
- filePath,
- originTestPath,
- optionsFromFile,
- hasSyntaxError,
- hasCompileError,
- hasRuntimeError,
- isNegativeIfChecked,
- hasCompileErrorIfChecked,
- hasStaticWarning,
+ var info = new TestInformation(filePath, originTestPath, optionsFromFile,
+ hasSyntaxError, hasCompileError, hasRuntimeError, hasStaticWarning,
multitestKey: multitestKey);
cachedTests.add(info);
enqueueTestCaseFromTestInformation(info);
