| // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'dart:async'; |
| import 'dart:convert'; |
| import 'dart:io' show pid; |
| |
| import 'package:path/path.dart' as p; |
| |
| import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports |
| import 'package:test_api/src/backend/group_entry.dart'; // ignore: implementation_imports |
| import 'package:test_api/src/backend/live_test.dart'; // ignore: implementation_imports |
| import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports |
| import 'package:test_api/src/backend/state.dart'; // ignore: implementation_imports |
| import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports |
| import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports |
| import 'package:test_api/src/frontend/expect.dart'; // ignore: implementation_imports |
| |
| import '../runner_suite.dart'; |
| import '../suite.dart'; |
| import '../configuration.dart'; |
| import '../engine.dart'; |
| import '../load_suite.dart'; |
| import '../reporter.dart'; |
| import '../version.dart'; |
| |
| /// A reporter that prints machine-readable JSON-formatted test results. |
| class JsonReporter implements Reporter { |
| /// The global configuration that applies to this reporter. |
| final Configuration _config; |
| |
| /// The engine used to run the tests. |
| final Engine _engine; |
| |
| final StringSink _sink; |
| |
| /// A stopwatch that tracks the duration of the full run. |
| final _stopwatch = Stopwatch(); |
| |
| /// Whether we've started [_stopwatch]. |
| /// |
| /// We can't just use `_stopwatch.isRunning` because the stopwatch is stopped |
| /// when the reporter is paused. |
| var _stopwatchStarted = false; |
| |
| /// Whether the reporter is paused. |
| var _paused = false; |
| |
| /// The set of all subscriptions to various streams. |
| final _subscriptions = <StreamSubscription>{}; |
| |
| /// An expando that associates unique IDs with [LiveTest]s. |
| final _liveTestIDs = <LiveTest, int>{}; |
| |
| /// An expando that associates unique IDs with [Suite]s. |
| final _suiteIDs = <Suite, int>{}; |
| |
| /// An expando that associates unique IDs with [Group]s. |
| final _groupIDs = <Group, int>{}; |
| |
| /// The next ID to associate with a [LiveTest]. |
| var _nextID = 0; |
| |
| /// Watches the tests run by [engine] and prints their results as JSON. |
| static JsonReporter watch(Engine engine, StringSink sink) => |
| JsonReporter._(engine, sink); |
| |
| JsonReporter._(this._engine, this._sink) : _config = Configuration.current { |
| _subscriptions.add(_engine.onTestStarted.listen(_onTestStarted)); |
| |
| /// Convert the future to a stream so that the subscription can be paused or |
| /// canceled. |
| _subscriptions.add(_engine.success.asStream().listen(_onDone)); |
| |
| _subscriptions.add(_engine.onSuiteAdded.listen(null, onDone: () { |
| _emit('allSuites', {'count': _engine.addedSuites.length}); |
| })); |
| |
| _emit('start', |
| {'protocolVersion': '0.1.1', 'runnerVersion': testVersion, 'pid': pid}); |
| } |
| |
| @override |
| void pause() { |
| if (_paused) return; |
| _paused = true; |
| |
| _stopwatch.stop(); |
| |
| for (var subscription in _subscriptions) { |
| subscription.pause(); |
| } |
| } |
| |
| @override |
| void resume() { |
| if (!_paused) return; |
| _paused = false; |
| |
| if (_stopwatchStarted) _stopwatch.start(); |
| |
| for (var subscription in _subscriptions) { |
| subscription.resume(); |
| } |
| } |
| |
| @override |
| void cancel() { |
| for (var subscription in _subscriptions) { |
| subscription.cancel(); |
| } |
| _subscriptions.clear(); |
| } |
| |
| /// A callback called when the engine begins running [liveTest]. |
| void _onTestStarted(LiveTest liveTest) { |
| if (!_stopwatchStarted) { |
| _stopwatchStarted = true; |
| _stopwatch.start(); |
| } |
| |
| var suiteID = _idForSuite(liveTest.suite); |
| |
| // Don't emit groups for load suites. They're always empty and they provide |
| // unnecessary clutter. |
| var groupIDs = liveTest.suite is LoadSuite |
| ? [] |
| : _idsForGroups(liveTest.groups, liveTest.suite); |
| |
| var suiteConfig = _configFor(liveTest.suite); |
| var id = _nextID++; |
| _liveTestIDs[liveTest] = id; |
| _emit('testStart', { |
| 'test': _addFrameInfo( |
| suiteConfig, |
| { |
| 'id': id, |
| 'name': liveTest.test.name, |
| 'suiteID': suiteID, |
| 'groupIDs': groupIDs, |
| 'metadata': _serializeMetadata(suiteConfig, liveTest.test.metadata) |
| }, |
| liveTest.test, |
| liveTest.suite.platform.runtime, |
| liveTest.suite.path) |
| }); |
| |
| /// Convert the future to a stream so that the subscription can be paused or |
| /// canceled. |
| _subscriptions.add( |
| liveTest.onComplete.asStream().listen((_) => _onComplete(liveTest))); |
| |
| _subscriptions.add(liveTest.onError |
| .listen((error) => _onError(liveTest, error.error, error.stackTrace))); |
| |
| _subscriptions.add(liveTest.onMessage.listen((message) { |
| _emit('print', { |
| 'testID': id, |
| 'messageType': message.type.name, |
| 'message': message.text |
| }); |
| })); |
| } |
| |
| /// Returns an ID for [suite]. |
| /// |
| /// If [suite] doesn't have an ID yet, this assigns one and emits a new event |
| /// for that suite. |
| int _idForSuite(Suite suite) { |
| if (_suiteIDs.containsKey(suite)) return _suiteIDs[suite]; |
| |
| var id = _nextID++; |
| _suiteIDs[suite] = id; |
| |
| // Give the load suite's suite the same ID, because it doesn't have any |
| // different metadata. |
| if (suite is LoadSuite) { |
| suite.suite.then((runnerSuite) { |
| _suiteIDs[runnerSuite] = id; |
| if (!_config.debug) return; |
| |
| // TODO(nweiz): test this when we have a library for communicating with |
| // the Chrome remote debugger, or when we have VM debug support. |
| _emit('debug', { |
| 'suiteID': id, |
| 'observatory': runnerSuite.environment.observatoryUrl?.toString(), |
| 'remoteDebugger': |
| runnerSuite.environment.remoteDebuggerUrl?.toString(), |
| }); |
| }); |
| } |
| |
| _emit('suite', { |
| 'suite': { |
| 'id': id, |
| 'platform': suite.platform.runtime.identifier, |
| 'path': suite.path |
| } |
| }); |
| return id; |
| } |
| |
| /// Returns a list of the IDs for all the groups in [groups], which are |
| /// contained in the suite identified by [suiteID]. |
| /// |
| /// If a group doesn't have an ID yet, this assigns one and emits a new event |
| /// for that group. |
| List<int> _idsForGroups(Iterable<Group> groups, Suite suite) { |
| int parentID; |
| return groups.map((group) { |
| if (_groupIDs.containsKey(group)) { |
| parentID = _groupIDs[group]; |
| return parentID; |
| } |
| |
| var id = _nextID++; |
| _groupIDs[group] = id; |
| |
| var suiteConfig = _configFor(suite); |
| _emit('group', { |
| 'group': _addFrameInfo( |
| suiteConfig, |
| { |
| 'id': id, |
| 'suiteID': _idForSuite(suite), |
| 'parentID': parentID, |
| 'name': group.name, |
| 'metadata': _serializeMetadata(suiteConfig, group.metadata), |
| 'testCount': group.testCount |
| }, |
| group, |
| suite.platform.runtime, |
| suite.path) |
| }); |
| parentID = id; |
| return id; |
| }).toList(); |
| } |
| |
| /// Serializes [metadata] into a JSON-protocol-compatible map. |
| Map _serializeMetadata(SuiteConfiguration suiteConfig, Metadata metadata) => |
| suiteConfig.runSkipped |
| ? {'skip': false, 'skipReason': null} |
| : {'skip': metadata.skip, 'skipReason': metadata.skipReason}; |
| |
| /// A callback called when [liveTest] finishes running. |
| void _onComplete(LiveTest liveTest) { |
| _emit('testDone', { |
| 'testID': _liveTestIDs[liveTest], |
| // For backwards-compatibility, report skipped tests as successes. |
| 'result': liveTest.state.result == Result.skipped |
| ? 'success' |
| : liveTest.state.result.toString(), |
| 'skipped': liveTest.state.result == Result.skipped, |
| 'hidden': !_engine.liveTests.contains(liveTest) |
| }); |
| } |
| |
| /// A callback called when [liveTest] throws [error]. |
| void _onError(LiveTest liveTest, error, StackTrace stackTrace) { |
| _emit('error', { |
| 'testID': _liveTestIDs[liveTest], |
| 'error': error.toString(), |
| 'stackTrace': '$stackTrace', |
| 'isFailure': error is TestFailure |
| }); |
| } |
| |
| /// A callback called when the engine is finished running tests. |
| /// |
| /// [success] will be `true` if all tests passed, `false` if some tests |
| /// failed, and `null` if the engine was closed prematurely. |
| void _onDone(bool success) { |
| cancel(); |
| _stopwatch.stop(); |
| |
| _emit('done', {'success': success}); |
| } |
| |
| /// Returns the configuration for [suite]. |
| /// |
| /// If [suite] is a [RunnerSuite], this returns [RunnerSuite.config]. |
| /// Otherwise, it returns [SuiteConfiguration.empty]. |
| SuiteConfiguration _configFor(Suite suite) => |
| suite is RunnerSuite ? suite.config : SuiteConfiguration.empty; |
| |
| /// Emits an event with the given type and attributes. |
| void _emit(String type, Map attributes) { |
| attributes['type'] = type; |
| attributes['time'] = _stopwatch.elapsed.inMilliseconds; |
| _sink.writeln(jsonEncode(attributes)); |
| } |
| |
| /// Modifies [map] to include line, column, and URL information from the first |
| /// frame of [entry.trace], as well as the first line in the original file. |
| /// |
| /// Returns [map]. |
| Map<String, dynamic> _addFrameInfo( |
| SuiteConfiguration suiteConfig, |
| Map<String, dynamic> map, |
| GroupEntry entry, |
| Runtime runtime, |
| String suitePath) { |
| var frame = entry.trace?.frames?.first; |
| var rootFrame = entry.trace?.frames?.firstWhere( |
| (frame) => |
| frame.uri.scheme == 'file' && |
| frame.uri.toFilePath() == p.absolute(suitePath), |
| orElse: () => null); |
| if (suiteConfig.jsTrace && runtime.isJS) { |
| frame = null; |
| rootFrame = null; |
| } |
| |
| map['line'] = frame?.line; |
| map['column'] = frame?.column; |
| map['url'] = frame?.uri?.toString(); |
| if (rootFrame != null && rootFrame != frame) { |
| map['root_line'] = rootFrame.line; |
| map['root_column'] = rootFrame.column; |
| map['root_url'] = rootFrame.uri.toString(); |
| } |
| return map; |
| } |
| } |