| // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'package:status_file/canonical_status_file.dart'; |
| |
| import 'result_json_models.dart'; |
| import '../results/configurations.dart'; |
| import '../results/status_expectations.dart'; |
| import '../results/status_files.dart'; |
| import '../util.dart'; |
| |
| typedef SectionsSuggestion ComputeSectionsFunc( |
| List<Configuration> failingConfigurations, |
| List<Configuration> passingConfigurations, |
| StatusExpectations expectations, |
| String testName); |
| |
| /// [FailingTest] captures the essential information of a failing test, to |
| /// suggest sections to be updated. |
| class FailingTest { |
| /// [result] holds data from the actual running of the test, such as name and |
| /// outcome. |
| final Result result; |
| |
| /// [TestResult] is the combined result for all tests in all configurations, |
| /// gathered from the result.logs. |
| final TestResult testResult; |
| final List<Configuration> failingConfigurations; |
| final List<Configuration> passingConfigurations; |
| |
| FailingTest(this.result, this.testResult, this.failingConfigurations, |
| this.passingConfigurations); |
| |
| // There are multiple strategies for finding candidate sections, which will |
| // then be presented to the user. |
| List<SectionsSuggestion> computeSections(StatusExpectations expectations) { |
| var computeSectionStrategies = <ComputeSectionsFunc>[ |
| _findExistingFailingEntriesStrategy, |
| _failingSectionsIntersectionStrategy, |
| _statusSectionDifferenceStrategy |
| ]; |
| |
| // We remember all the sections already found by previous strategies. The |
| // most precise strategies should be called first, so a section is printed |
| // with the most accurate description of why it is a good candidate.. |
| var seenSections = new Set<StatusSectionWithFile>(); |
| |
| var computedList = <SectionsSuggestion>[]; |
| computeSectionStrategies.forEach((strategy) { |
| var suggestion = strategy(failingConfigurations, passingConfigurations, |
| expectations, result.name); |
| suggestion.sections = suggestion.sections |
| .where((section) => !seenSections.contains(section)) |
| .toList(); |
| if (suggestion != null && suggestion.sections.isNotEmpty) { |
| computedList.add(suggestion); |
| seenSections.addAll(suggestion.sections); |
| } |
| }); |
| return computedList; |
| } |
| |
| /// Checks, from [expectations], if the failing test is still failing. |
| bool stillFailing(StatusExpectations expectations) { |
| var testExpectations = expectations.getTestResultsWithExpectation(); |
| return testExpectations |
| .where((expectation) => |
| !expectation.isSuccess() && expectation.result == result) |
| .isNotEmpty; |
| } |
| |
| /// Gets the status files for the suite this test belongs to. |
| List<StatusFile> statusFiles(StatusExpectations expectations) { |
| var testSuite = getSuiteNameForTest(result.name); |
| return expectations.statusFilesMaps[testSuite].statusFiles; |
| } |
| |
| /// Gets all failing status entries. Test can still fail without having any |
| /// entries. |
| List<StatusSectionEntry> failingStatusEntries( |
| StatusExpectations expectations) { |
| var entries = _sectionEntriesForTestInConfigurations( |
| expectations, failingConfigurations, result.name, |
| success: false); |
| return new Set.of(entries).toList(); |
| } |
| |
| /// Gets the failing configurations not covered by expressions in [sections]. |
| List<Configuration> failingConfigurationsNotCovered( |
| StatusExpectations expectations, List<StatusSectionWithFile> sections) { |
| return _configurationsNotCovered( |
| expectations, sections, this.failingConfigurations); |
| } |
| |
| /// Gets the passing configurations not covered by expressions in [sections]. |
| List<Configuration> passingConfigurationsNotCovered( |
| StatusExpectations expectations, List<StatusSectionWithFile> sections) { |
| return _configurationsNotCovered( |
| expectations, sections, this.passingConfigurations); |
| } |
| |
| List<Configuration> _configurationsNotCovered( |
| StatusExpectations expectations, |
| List<StatusSectionWithFile> sections, |
| List<Configuration> configurations) { |
| List<Configuration> notCovered = []; |
| for (var configuration in configurations) { |
| var environment = expectations.configurationEnvironments[configuration]; |
| if (!sections |
| .any((section) => section.section.condition.evaluate(environment))) { |
| notCovered.add(configuration); |
| } |
| } |
| return notCovered; |
| } |
| } |
| |
| /// Finds status section entries (entries with a path and expectation), where |
| /// the path matches the test name, in [statusExpectations] for the |
| /// [configurations]. [success] can be used to further filter on the entries, by |
| /// specifying if the status section entry should be successful, failing or both |
| /// (null), when compared with the result from the test, in one of the |
| /// configurations. |
| List<StatusSectionEntry> _sectionEntriesForTestInConfigurations( |
| StatusExpectations statusExpectations, |
| List<Configuration> configurations, |
| String testName, |
| {bool success: null}) { |
| var expectations = statusExpectations.getTestResultsWithExpectation(); |
| return expectations |
| .where((expectation) => |
| configurations.contains(expectation.configuration) && |
| expectation.result.name == testName && |
| (success == null || expectation.isSuccess() == success)) |
| .expand((testResult) => testResult.entries) |
| .toList(); |
| } |
| |
| /********************************* |
| * Strategies to find sections |
| *********************************/ |
| |
| /// Strategy that returns all status headers that have a failing status entry |
| /// for this test. For tests going from some failure to passing, this would be |
| /// the preferred option. |
| /// We are not guaranteed that any sections will cover all failing sections. |
| SectionsSuggestion _findExistingFailingEntriesStrategy( |
| List<Configuration> failingConfigurations, |
| List<Configuration> passingConfigurations, |
| StatusExpectations expectations, |
| String testName) { |
| String description = "These sections already already have entries for this " |
| "test, that apply to at least one failing configuration"; |
| var allEntries = new Set<StatusSectionEntry>.from( |
| _sectionEntriesForTestInConfigurations( |
| expectations, failingConfigurations, testName, |
| success: false)); |
| var sections = allEntries |
| .map( |
| (entry) => new StatusSectionWithFile(entry.statusFile, entry.section)) |
| .toList(); |
| return new SectionsSuggestion(description, sections); |
| } |
| |
| /// The [_failingSectionsIntersectionStrategy] will hopefully be the bread and |
| /// butter strategy for most. It takes the intersection of all failing |
| /// configurations enabled sections and subtracts the set of all sections |
| /// enabled for the passing configurations. |
| /// If a result is found, it is guaranteed to cover all the failing |
| /// configurations and none of the passing configurations. |
| SectionsSuggestion _failingSectionsIntersectionStrategy( |
| List<Configuration> failingConfigurations, |
| List<Configuration> passingConfigurations, |
| StatusExpectations expectations, |
| String testName) { |
| String description = "Every section listed here covers all failing" |
| "configurations and none of the passing configurations."; |
| var testSuite = getSuiteNameForTest(testName); |
| var configurationSections = <StatusSectionWithFile, List<Configuration>>{}; |
| for (var configuration in failingConfigurations) { |
| var sections = |
| _sectionsFromConfiguration(expectations, configuration, testSuite); |
| for (var section in _filterSections(sections, testSuite, configuration)) { |
| configurationSections.putIfAbsent(section, () => [])..add(configuration); |
| } |
| } |
| var sections = new Set<StatusSectionWithFile>.from(configurationSections.keys |
| .where((key) => |
| failingConfigurations.length == configurationSections[key].length)); |
| if (sections.isEmpty) { |
| return new SectionsSuggestion(description, []); |
| } |
| for (var configuration in passingConfigurations) { |
| var sectionsToRemove = |
| _sectionsFromConfiguration(expectations, configuration, testSuite); |
| sections.removeAll(sectionsToRemove); |
| } |
| var sortedSections = sections.toList() |
| ..sort((a, b) => a.section.condition.compareTo(b.section.condition)); |
| return new SectionsSuggestion(description, sortedSections); |
| } |
| |
| /// The [_statusSectionDifferenceStrategy] takes the union of all enabled |
| /// failing sections and subtracts all the enabled passing sections. We are |
| /// guaranteed to not select a section that is enabled in a passing |
| /// configuration, however, it is not guaranteed that a combination of the |
| /// sections will cover all failing configurations. |
| SectionsSuggestion _statusSectionDifferenceStrategy( |
| List<Configuration> failingConfigurations, |
| List<Configuration> passingConfigurations, |
| StatusExpectations expectations, |
| String testName) { |
| String description = "All sections cover one or more failing " |
| "configuration but none of the passing configurations."; |
| var configurationSections = <Configuration, Set<StatusSectionWithFile>>{}; |
| var testSuite = getSuiteNameForTest(testName); |
| for (var configuration in failingConfigurations) { |
| var sections = _filterSections( |
| _sectionsFromConfiguration(expectations, configuration, testSuite), |
| testSuite, |
| configuration); |
| configurationSections[configuration] = new Set.from(sections); |
| } |
| for (var configuration in passingConfigurations) { |
| var sectionsToRemove = |
| _sectionsFromConfiguration(expectations, configuration, testSuite); |
| for (var failingSections in configurationSections.values) { |
| failingSections.removeAll(sectionsToRemove); |
| } |
| } |
| var sortedSections = configurationSections.values |
| .reduce((a, b) => a..addAll(b)) |
| .toList() |
| ..sort((a, b) => a.section.condition.compareTo(b.section.condition)); |
| return new SectionsSuggestion(description, sortedSections); |
| } |
| |
| List<StatusSectionWithFile> _sectionsFromConfiguration( |
| StatusExpectations expectations, |
| Configuration configuration, |
| String testSuite) { |
| var environment = expectations.configurationEnvironments[configuration]; |
| return expectations.statusFilesMaps[testSuite] |
| .sectionsForConfiguration(environment); |
| } |
| |
| /// Filters section by not taking the default section, and also exclude status |
| /// files of other compilers. |
| List<StatusSectionWithFile> _filterSections( |
| List<StatusSectionWithFile> statusSections, |
| String suite, |
| Configuration configuration) { |
| String specificStatusFile = |
| configuration.compiler == Compiler.none ? "" : configuration.compiler; |
| if (configuration.compiler == Compiler.dartdevk.name) { |
| specificStatusFile = Compiler.dartdevc.name; |
| } |
| if (configuration.compiler == Compiler.dartk || |
| configuration.compiler == Compiler.dartkp) { |
| specificStatusFile = "kernel"; |
| } |
| if (specificStatusFile.isEmpty) { |
| return statusSections |
| .where((statusSection) => statusSection.section.condition != null); |
| } else { |
| return statusSections |
| .where((statusSection) => |
| statusSection.section.condition != null && |
| (statusSection.statusFile.path.endsWith("$suite.status") || |
| statusSection.statusFile.path |
| .endsWith("${suite}-$specificStatusFile.status") || |
| statusSection.statusFile.path |
| .endsWith("${suite}_$specificStatusFile.status"))) |
| .toList(); |
| } |
| } |
| |
| /// A [SectionsSuggestion] object holds all the sections suggested by a |
| /// strategy, and a description of that strategy. |
| class SectionsSuggestion { |
| final String strategy; |
| List<StatusSectionWithFile> sections; |
| SectionsSuggestion(this.strategy, this.sections); |
| } |