| // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'dart:io'; |
| import 'dart:async'; |
| import 'package:gardening/src/results/configuration_environment.dart'; |
| import 'package:gardening/src/results/status_files.dart'; |
| |
| import 'result_json_models.dart'; |
| import 'testpy_wrapper.dart'; |
| import '../util.dart'; |
| import 'package:status_file/expectation.dart'; |
| |
| class StatusExpectations { |
| final TestResult testResult; |
| |
| final Map<Configuration, ConfigurationEnvironment> configurationEnvironments = |
| {}; |
| final Map<String, StatusFiles> statusFilesMaps = {}; |
| |
| bool _isLoaded = false; |
| |
| StatusExpectations(this.testResult); |
| |
| /// Build expectations from configurations. Each configuration may test |
| /// multiple test suites. |
| Future loadStatusFiles() async { |
| if (_isLoaded) { |
| return; |
| } |
| |
| Map<String, Iterable<String>> suiteStatusFiles = {}; |
| |
| await Future.wait(testResult.configurations.keys.map((key) async { |
| Configuration configuration = testResult.configurations[key]; |
| if (!configurationEnvironments.containsKey(key)) { |
| configurationEnvironments[configuration] = |
| new ConfigurationEnvironment(configuration); |
| } |
| // We allow overwriting, since status files for a suite does not change. |
| suiteStatusFiles.addAll(await statusFileListerMap(configuration)); |
| })); |
| |
| suiteStatusFiles.keys.forEach((suite) { |
| var statusFilePaths = suiteStatusFiles[suite].map((file) { |
| return "${PathHelper.sdkRepositoryRoot()}/$file"; |
| }).where((sf) { |
| return new File(sf).existsSync(); |
| }).toList(); |
| statusFilesMaps[suite] = StatusFiles.read(statusFilePaths); |
| }); |
| |
| _isLoaded = true; |
| } |
| |
| /// Finds the expectation for each test found in the [testResult] results and |
| /// outputs if the test succeeded or failed. The status files must have been |
| /// loaded. |
| List<TestExpectationResult> getTestResultsWithExpectation() { |
| assert(_isLoaded); |
| List<TestExpectationResult> expectationResults = []; |
| testResult.results.forEach((result) { |
| try { |
| Configuration configuration = |
| testResult.configurations[result.configuration]; |
| ConfigurationEnvironment environment = |
| configurationEnvironments[configuration]; |
| var testSuite = getSuiteNameForTest(result.name); |
| StatusFiles expectationSuite = statusFilesMaps[testSuite]; |
| var qualifiedName = getQualifiedNameForTest(result.name); |
| var statusFileEntries = expectationSuite |
| .sectionsWithTestForConfiguration(environment, qualifiedName); |
| expectationResults.add(new TestExpectationResult( |
| statusFileEntries, result, configuration)); |
| } catch (ex) {} |
| }); |
| |
| return expectationResults; |
| } |
| } |
| |
| /// [TestExpectationResult] contains information about the result of running a |
| /// test, along with the expectation. |
| class TestExpectationResult { |
| final List<StatusSectionEntry> entries; |
| final Result result; |
| final Configuration configuration; |
| |
| TestExpectationResult(this.entries, this.result, this.configuration); |
| |
| bool _isSuccess; |
| |
| /// Gets all expectations from status file entries as a set. |
| Set<Expectation> expectations() { |
| Set<Expectation> expectationsFromEntries = entries |
| .map((expectation) => expectation.entry.expectations) |
| .expand((expectation) => expectation) |
| .toSet(); |
| if (expectationsFromEntries.isEmpty) { |
| expectationsFromEntries.add(Expectation.pass); |
| } |
| return expectationsFromEntries; |
| } |
| |
| /// Determines if a result matches its expectation. |
| bool isSuccess() { |
| if (_isSuccess != null) { |
| return _isSuccess; |
| } |
| Expectation outcome = Expectation.find(result.result); |
| Set<Expectation> testExpectations = |
| expectationsFromTest(result.testExpectations); |
| Set<Expectation> expectationSet = expectations(); |
| final allowedWithoutRuntime = [ |
| Expectation.runtimeError, |
| Expectation.missingRuntimeError, |
| Expectation.timeout |
| ]; |
| _isSuccess = testExpectations.contains(outcome) || |
| expectationSet.contains(Expectation.skip) || |
| expectationSet.contains(Expectation.skipByDesign) || |
| outcome == Expectation.pass && |
| configuration.runtime == "none" && |
| allowedWithoutRuntime.any(expectationSet.contains) || |
| expectationSet.any((expectation) { |
| return outcome.canBeOutcomeOf(expectation); |
| }); |
| return _isSuccess; |
| } |
| } |
| |
| Set<Expectation> expectationsFromTest(List<String> testExpectations) { |
| if (testExpectations == null) { |
| return new Set<Expectation>(); |
| } |
| return testExpectations.map((exp) { |
| if (exp == "static-type-warning") { |
| return Expectation.staticWarning; |
| } else if (exp == "runtime-error") { |
| return Expectation.runtimeError; |
| } else if (exp == "compile-time-error") { |
| return Expectation.compileTimeError; |
| } |
| return null; |
| }).toSet(); |
| } |