| // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| @TestOn('vm') |
| |
| import 'dart:io'; |
| import 'dart:math' as math; |
| |
| import 'package:test/test.dart'; |
| import 'package:test_core/src/util/exit_codes.dart' as exit_codes; |
| import 'package:test_descriptor/test_descriptor.dart' as d; |
| |
| import '../io.dart'; |
| |
| final _success = ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("success", () {}); |
| } |
| '''; |
| |
| final _failure = ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("failure", () => throw TestFailure("oh no")); |
| } |
| '''; |
| |
| final _asyncFailure = ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("failure", () async { |
| await Future(() {}).then((_) { |
| throw 'oh no'; |
| }); |
| }); |
| } |
| '''; |
| |
| final _defaultConcurrency = math.max(1, Platform.numberOfProcessors ~/ 2); |
| |
| final _usage = ''' |
| Usage: pub run test [files or directories...] |
| |
| -h, --help Shows this usage information. |
| --version Shows the package's version. |
| |
| ======== Selecting Tests |
| -n, --name A substring of the name of the test to run. |
| Regular expression syntax is supported. |
| If passed multiple times, tests must match all substrings. |
| |
| -N, --plain-name A plain-text substring of the name of the test to run. |
| If passed multiple times, tests must match all substrings. |
| |
| -t, --tags Run only tests with all of the specified tags. |
| Supports boolean selector syntax. |
| |
| -x, --exclude-tags Don't run tests with any of the specified tags. |
| Supports boolean selector syntax. |
| |
| --[no-]run-skipped Run skipped tests instead of skipping them. |
| |
| ======== Running Tests |
| -p, --platform The platform(s) on which to run the tests. |
| $_browsers |
| |
| -P, --preset The configuration preset(s) to use. |
| -j, --concurrency=<threads> The number of concurrent test suites run. |
| (defaults to "$_defaultConcurrency") |
| |
| --total-shards The total number of invocations of the test runner being run. |
| --shard-index The index of this test runner invocation (of --total-shards). |
| --pub-serve=<port> The port of a pub serve instance serving "test/". |
| --timeout The default test timeout. For example: 15s, 2x, none |
| (defaults to "30s") |
| |
| --pause-after-load Pauses for debugging before any tests execute. |
| Implies --concurrency=1, --debug, and --timeout=none. |
| Currently only supported for browser tests. |
| |
| --debug Runs the VM and Chrome tests in debug mode. |
| --coverage=<directory> Gathers coverage and outputs it to the specified directory. |
| Implies --debug. |
| |
| --[no-]chain-stack-traces Chained stack traces to provide greater exception details |
| especially for asynchronous code. It may be useful to disable |
| to provide improved test performance but at the cost of |
| debuggability. |
| (defaults to on) |
| |
| --no-retry Don't re-run tests that have retry set. |
| --test-randomize-ordering-seed The seed to randomize the execution order of test cases. |
| Must be a 32bit unsigned integer or "random". |
| If "random", pick a random seed to use. |
| If not passed, do not randomize test case execution order. |
| |
| ======== Output |
| -r, --reporter The runner used to print test results. |
| |
| [compact] A single line, updated continuously. |
| [expanded] (default) A separate line for each update. |
| [json] A machine-readable format (see https://goo.gl/gBsV1a). |
| |
| --file-reporter The reporter used to write test results to a file. |
| Should be in the form <reporter>:<filepath>, e.g. "json:reports/tests.json" |
| |
| --verbose-trace Whether to emit stack traces with core library frames. |
| --js-trace Whether to emit raw JavaScript stack traces for browser tests. |
| --[no-]color Whether to use terminal colors. |
| (auto-detected by default) |
| '''; |
| |
| final _browsers = '[vm (default), chrome, phantomjs, firefox' + |
| (Platform.isMacOS ? ', safari' : '') + |
| (Platform.isWindows ? ', ie' : '') + |
| ', node]'; |
| |
| void main() { |
| test('prints help information', () async { |
| var test = await runTest(['--help']); |
| expectStdoutEquals(test, ''' |
| Runs tests in this package. |
| |
| $_usage'''); |
| await test.shouldExit(0); |
| }); |
| |
| group('fails gracefully if', () { |
| test('an invalid option is passed', () async { |
| var test = await runTest(['--asdf']); |
| expectStderrEquals(test, ''' |
| Could not find an option named "asdf". |
| |
| $_usage'''); |
| await test.shouldExit(exit_codes.usage); |
| }); |
| |
| test('a non-existent file is passed', () async { |
| var test = await runTest(['file']); |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading file [E]', |
| 'Failed to load "file": Does not exist.' |
| ])); |
| await test.shouldExit(1); |
| }); |
| |
| test("the default directory doesn't exist", () async { |
| var test = await runTest([]); |
| expectStderrEquals(test, ''' |
| No test files were passed and the default "test/" directory doesn't exist. |
| |
| $_usage'''); |
| await test.shouldExit(exit_codes.data); |
| }); |
| |
| test('a test file fails to load', () async { |
| await d.file('test.dart', 'invalid Dart file').create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| 'Failed to load "test.dart":', |
| 'Unable to spawn isolate: test.dart:1:9: Error: ' |
| "Expected ';' after this.", |
| 'invalid Dart file' |
| ])); |
| |
| await test.shouldExit(1); |
| }); |
| |
| // This syntax error is detected lazily, and so requires some extra |
| // machinery to support. |
| test('a test file fails to parse due to a missing semicolon', () async { |
| await d.file('test.dart', 'void main() {foo}').create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading test.dart [E]', |
| 'Failed to load "test.dart":', |
| 'Unable to spawn isolate: test.dart:1:14: ' |
| "Error: Expected ';' after this" |
| ])); |
| |
| await test.shouldExit(1); |
| }); |
| |
| // This is slightly different from the above test because it's an error |
| // that's caught first by the analyzer when it's used to parse the file. |
| test('a test file fails to parse', () async { |
| await d.file('test.dart', '@TestOn)').create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading test.dart [E]', |
| 'Failed to load "test.dart":', |
| 'Unable to spawn isolate: test.dart:1:8: Error: ' |
| "Expected a declaration, but got ')'", |
| '@TestOn)', |
| ])); |
| |
| await test.shouldExit(1); |
| }); |
| |
| test("an annotation's contents are invalid", () async { |
| await d.file('test.dart', "@TestOn('zim')\nlibrary foo;").create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading test.dart [E]', |
| 'Failed to load "test.dart":', |
| 'Error on line 1, column 10: Undefined variable.', |
| "@TestOn('zim')", |
| ' ^^^' |
| ])); |
| await test.shouldExit(1); |
| }); |
| |
| test('a test file throws', () async { |
| await d.file('test.dart', "void main() => throw 'oh no';").create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading test.dart [E]', |
| 'Failed to load "test.dart": oh no' |
| ])); |
| await test.shouldExit(1); |
| }); |
| |
| test("a test file doesn't have a main defined", () async { |
| await d.file('test.dart', 'void foo() {}').create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading test.dart [E]', |
| "Error: Getter not found: 'main'", |
| ])); |
| |
| await test.shouldExit(1); |
| }); |
| |
| test('a test file has a non-function main', () async { |
| await d.file('test.dart', 'int main;').create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading test.dart [E]', |
| "A value of type 'int' can't be assigned to a " |
| "variable of type 'Function'", |
| ])); |
| |
| await test.shouldExit(1); |
| }); |
| |
| test('a test file has a main with arguments', () async { |
| await d.file('test.dart', 'void main(arg) {}').create(); |
| var test = await runTest(['test.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading test.dart [E]', |
| 'Failed to load "test.dart": Top-level main() function takes arguments.' |
| ])); |
| await test.shouldExit(1); |
| }); |
| |
| test('multiple load errors occur', () async { |
| await d.file('test.dart', 'invalid Dart file').create(); |
| var test = await runTest(['test.dart', 'nonexistent.dart']); |
| |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '-1: loading nonexistent.dart [E]', |
| 'Failed to load "nonexistent.dart": Does not exist', |
| '-2: loading test.dart [E]', |
| 'Failed to load "test.dart"', |
| ])); |
| |
| await test.shouldExit(1); |
| }); |
| |
| // TODO(nweiz): test what happens when a test file is unreadable once issue |
| // 15078 is fixed. |
| }); |
| |
| group('runs successful tests', () { |
| test('defined in a single file', () async { |
| await d.file('test.dart', _success).create(); |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('defined in a directory', () async { |
| for (var i = 0; i < 3; i++) { |
| await d.file('${i}_test.dart', _success).create(); |
| } |
| |
| var test = await runTest(['.']); |
| expect(test.stdout, emitsThrough(contains('+3: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('defaulting to the test directory', () async { |
| await d |
| .dir( |
| 'test', |
| Iterable.generate(3, (i) { |
| return d.file('${i}_test.dart', _success); |
| })) |
| .create(); |
| |
| var test = await runTest([]); |
| expect(test.stdout, emitsThrough(contains('+3: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('directly', () async { |
| await d.file('test.dart', _success).create(); |
| var test = await runDart(['test.dart']); |
| |
| expect(test.stdout, emitsThrough(contains('All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| |
| // Regression test; this broke in 0.12.0-beta.9. |
| test('on a file in a subdirectory', () async { |
| await d.dir('dir', [d.file('test.dart', _success)]).create(); |
| |
| var test = await runTest(['dir/test.dart']); |
| expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| }); |
| |
| group('runs failing tests', () { |
| test('defaults to chaining stack traces', () async { |
| await d.file('test.dart', _asyncFailure).create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('asynchronous gap'))); |
| await test.shouldExit(1); |
| }); |
| |
| test('respects the chain-stack-traces flag', () async { |
| await d.file('test.dart', _asyncFailure).create(); |
| |
| var test = await runTest(['test.dart', '--no-chain-stack-traces']); |
| expect( |
| test.stdout, |
| containsInOrder([ |
| '00:00 +0: failure', |
| '00:00 +0 -1: failure [E]', |
| 'oh no', |
| 'test.dart 8:7 main.<fn>', |
| ])); |
| await test.shouldExit(1); |
| }); |
| |
| test('defined in a single file', () async { |
| await d.file('test.dart', _failure).create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('-1: Some tests failed.'))); |
| await test.shouldExit(1); |
| }); |
| |
| test('defined in a directory', () async { |
| for (var i = 0; i < 3; i++) { |
| await d.file('${i}_test.dart', _failure).create(); |
| } |
| |
| var test = await runTest(['.']); |
| expect(test.stdout, emitsThrough(contains('-3: Some tests failed.'))); |
| await test.shouldExit(1); |
| }); |
| |
| test('defaulting to the test directory', () async { |
| await d |
| .dir( |
| 'test', |
| Iterable.generate(3, (i) { |
| return d.file('${i}_test.dart', _failure); |
| })) |
| .create(); |
| |
| var test = await runTest([]); |
| expect(test.stdout, emitsThrough(contains('-3: Some tests failed.'))); |
| await test.shouldExit(1); |
| }); |
| |
| test('directly', () async { |
| await d.file('test.dart', _failure).create(); |
| var test = await runDart(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('Some tests failed.'))); |
| await test.shouldExit(255); |
| }); |
| }); |
| |
| test('runs tests even when a file fails to load', () async { |
| await d.file('test.dart', _success).create(); |
| |
| var test = await runTest(['test.dart', 'nonexistent.dart']); |
| expect(test.stdout, emitsThrough(contains('+1 -1: Some tests failed.'))); |
| await test.shouldExit(1); |
| }); |
| |
| group('with a top-level @Skip declaration', () { |
| setUp(() async { |
| await d.file('test.dart', ''' |
| @Skip() |
| |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("success", () {}); |
| } |
| ''').create(); |
| }); |
| |
| test('skips all tests', () async { |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+0 ~1: All tests skipped.'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('runs all tests with --run-skipped', () async { |
| var test = await runTest(['--run-skipped', 'test.dart']); |
| expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| }); |
| |
| group('with onPlatform', () { |
| test('respects matching Skips', () async { |
| await d.file('test.dart', ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("fail", () => throw 'oh no', onPlatform: {"vm": Skip()}); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+0 ~1: All tests skipped.'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('ignores non-matching Skips', () async { |
| await d.file('test.dart', ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("success", () {}, onPlatform: {"chrome": Skip()}); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('respects matching Timeouts', () async { |
| await d.file('test.dart', ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("fail", () async { |
| await Future.delayed(Duration.zero); |
| throw 'oh no'; |
| }, onPlatform: { |
| "vm": Timeout(Duration.zero) |
| }); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect( |
| test.stdout, |
| containsInOrder( |
| ['Test timed out after 0 seconds.', '-1: Some tests failed.'])); |
| await test.shouldExit(1); |
| }); |
| |
| test('ignores non-matching Timeouts', () async { |
| await d.file('test.dart', ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("success", () {}, onPlatform: { |
| "chrome": Timeout(Duration(seconds: 0)) |
| }); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('applies matching platforms in order', () async { |
| await d.file('test.dart', ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("success", () {}, onPlatform: { |
| "vm": Skip("first"), |
| "vm || windows": Skip("second"), |
| "vm || linux": Skip("third"), |
| "vm || mac-os": Skip("fourth"), |
| "vm || android": Skip("fifth") |
| }); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdoutStream(), neverEmits(contains('Skip: first'))); |
| expect(test.stdoutStream(), neverEmits(contains('Skip: second'))); |
| expect(test.stdoutStream(), neverEmits(contains('Skip: third'))); |
| expect(test.stdoutStream(), neverEmits(contains('Skip: fourth'))); |
| expect(test.stdout, emitsThrough(contains('Skip: fifth'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('applies platforms to a group', () async { |
| await d.file('test.dart', ''' |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| group("group", () { |
| test("success", () {}); |
| }, onPlatform: { |
| "vm": Skip() |
| }); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('All tests skipped.'))); |
| await test.shouldExit(0); |
| }); |
| }); |
| |
| group('with an @OnPlatform annotation', () { |
| test('respects matching Skips', () async { |
| await d.file('test.dart', ''' |
| @OnPlatform(const {"vm": const Skip()}) |
| |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("fail", () => throw 'oh no'); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+0 ~1: All tests skipped.'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('ignores non-matching Skips', () async { |
| await d.file('test.dart', ''' |
| @OnPlatform(const {"chrome": const Skip()}) |
| |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("success", () {}); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| |
| test('respects matching Timeouts', () async { |
| await d.file('test.dart', ''' |
| @OnPlatform(const { |
| "vm": const Timeout(const Duration(seconds: 0)) |
| }) |
| |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("fail", () async { |
| await Future.delayed(Duration.zero); |
| throw 'oh no'; |
| }); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect( |
| test.stdout, |
| containsInOrder( |
| ['Test timed out after 0 seconds.', '-1: Some tests failed.'])); |
| await test.shouldExit(1); |
| }); |
| |
| test('ignores non-matching Timeouts', () async { |
| await d.file('test.dart', ''' |
| @OnPlatform(const { |
| "chrome": const Timeout(const Duration(seconds: 0)) |
| }) |
| |
| import 'dart:async'; |
| |
| import 'package:test/test.dart'; |
| |
| void main() { |
| test("success", () {}); |
| } |
| ''').create(); |
| |
| var test = await runTest(['test.dart']); |
| expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); |
| await test.shouldExit(0); |
| }); |
| }); |
| |
| test('with the --color flag, uses colors', () async { |
| await d.file('test.dart', _failure).create(); |
| var test = await runTest(['--color', 'test.dart']); |
| // This is the color code for red. |
| expect(test.stdout, emitsThrough(contains('\u001b[31m'))); |
| await test.shouldExit(); |
| }); |
| |
| group('runs tests successfully more than once when calling runTests', () { |
| test('defined in a single file', () async { |
| await d.file('test.dart', _success).create(); |
| await d.file('runner.dart', ''' |
| import 'package:test_core/src/executable.dart' as test; |
| |
| void main(List<String> args) async { |
| await test.runTests(args); |
| await test.runTests(args); |
| test.completeShutdown(); |
| }''').create(); |
| var test = await runDart(['runner.dart', '--no-color', '--', 'test.dart'], |
| description: 'dart runner.dart -- test.dart', |
| environment: {'FORCE_TEST_EXIT': 'false'}); |
| expect( |
| test.stdout, |
| emitsThrough(containsInOrder([ |
| '+0: loading test.dart', |
| '+0: success', |
| '+1: success', |
| 'All tests passed!' |
| ]))); |
| expect( |
| test.stdout, |
| emitsThrough(containsInOrder([ |
| '+0: loading test.dart', |
| '+0: success', |
| '+1: success', |
| '+1: All tests passed!', |
| ]))); |
| await test.shouldExit(0); |
| }); |
| }); |
| } |