tools/gardening/bin/results_get.dart

// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'package:args/command_runner.dart';
import 'package:args/args.dart';
import 'package:gardening/src/results/status_expectations.dart';
import 'package:gardening/src/results/status_files.dart';
import 'package:gardening/src/results/test_result_helper.dart';
import 'package:gardening/src/results/test_result_service.dart';
import 'package:gardening/src/util.dart';
import 'package:gardening/src/console_table.dart';
import 'package:gardening/src/results/result_json_models.dart' as models;
import 'package:gardening/src/buildbucket.dart';
import 'package:gardening/src/extended_printer.dart';

/// Build standard arguments for input.
void buildArgs(ArgParser argParser) {
  argParser.addFlag('scripting',
      defaultsTo: false,
      abbr: 's',
      negatable: false,
      help: "Use flag to remove templated output.");
}

/// Get output table based on arguments passed in [argResults].
OutputTable getOutputTable(ArgResults argResults) {
  if (argResults["scripting"]) {
    return new ScriptTable();
  }
  return new ConsoleTable(template: rows);
}

String howToUse(String command) {
  return "Use by calling one of the following:\n\n"
      "\tget $command <file>                     : for a local result.log file.\n"
      "\tget $command <uri_to_result_log>        : for direct links to result.logs.\n"
      "\tget $command <uri_try_bot>              : for links to try bot builders.\n"
      "\tget $command <commit_number> <patchset> : for links to try bot builders.\n"
      "\tget $command <builder>                  : for a builder name.\n"
      "\tget $command <builder> <build>          : for a builder and build number.\n"
      "\tget $command <builder_group>            : for a builder group.\n";
}

/// [GetCommand] handles when given command 'get' and expect a sub-command.
class GetCommand extends Command {
  @override
  String get description => "Get results for files and test-suites.";

  @override
  String get name => "get";

  GetCommand() {
    addSubcommand(new GetTestsWithResultCommand());
    addSubcommand(new GetTestsWithResultAndExpectationCommand());
    addSubcommand(new GetTestFailuresCommand());
  }
}

/// [GetTestsWithResultCommand] handles when given the sub-command 'tests' and
/// returns a list of tests with their respective results.
class GetTestsWithResultCommand extends Command {
  @override
  String get description => "Get a list of tests with their respective "
      "results from result.logs found from input.";

  @override
  String get name => "tests";

  GetTestsWithResultCommand() {
    buildArgs(argParser);
  }

  Future run() async {
    models.TestResult testResults =
        await getTestResultFromBuilder(argResults.rest);
    if (testResults == null) {
      print(howToUse("tests"));
      return;
    }
    var outputTable = getOutputTable(argResults)
      ..addHeader(new Column("Test", width: 60), (item) {
        return item.name;
      })
      ..addHeader(new Column("Result"), (item) {
        return item.result;
      });
    outputTable.print(testResults.results);
  }
}

/// [GetTestsWithResultAndExpectationCommand] handles when given the sub-command
/// 'result' and returns a list of tests with their result and expectations.
class GetTestsWithResultAndExpectationCommand extends Command {
  @override
  String get description => "Get a list of tests with their respective "
      "results and expectations from result.logs found from input.";

  @override
  String get name => "tests-with-expectations";

  GetTestsWithResultAndExpectationCommand() {
    buildArgs(argParser);
  }

  Future run() async {
    models.TestResult testResult = null;

    if (isCqInput(argResults.rest)) {
      Iterable<BuildBucketTestResult> buildBucketTestResults =
          await getTestResultsFromCq(argResults.rest);
      if (buildBucketTestResults != null) {
        testResult = buildBucketTestResults.fold<models.TestResult>(
            new models.TestResult(),
            (combined, buildResult) => combined..combineWith([buildResult]));
      }
    } else {
      testResult = await getTestResultFromBuilder(argResults.rest);
    }

    if (testResult == null) {
      print(howToUse("results"));
      return;
    }

    var statusExpectations = new StatusExpectations(testResult);
    await statusExpectations.loadStatusFiles();
    List<TestExpectationResult> withExpectations =
        statusExpectations.getTestResultsWithExpectation();

    var outputTable = getOutputTable(argResults)
      ..addHeader(new Column("Test", width: 38), (item) {
        return item.result.name;
      })
      ..addHeader(new Column("Result", width: 18), (item) {
        return item.result.result;
      })
      ..addHeader(new Column("Expected"), (item) {
        return item.entries.toString();
      })
      ..addHeader(new Column("Success", width: 4), (item) {
        return item.isSuccess() ? "OK" : "FAIL";
      });
    outputTable.print(withExpectations);
  }
}

/// [GetTestFailuresCommand] handles when given the sub-command 'failures' and
/// returns only the failing tests.
class GetTestFailuresCommand extends Command {
  @override
  String get description => "Get a list of tests with their respective "
      "results and expectations from result.logs found from input.";

  @override
  String get name => "failures";

  GetTestFailuresCommand() {
    buildArgs(argParser);
  }

  Future run() async {
    List<models.TestResult> testResults = [];
    if (isCqInput(argResults.rest)) {
      var buildBucketResults = await getTestResultsFromCq(argResults.rest);
      if (buildBucketResults == null) {
        print(howToUse("failures"));
        return;
      }
      testResults.addAll(buildBucketResults);
    } else {
      var testResult = await getTestResultFromBuilder(argResults.rest);
      if (testResult == null) {
        print(howToUse("failures"));
        return;
      }
      testResults.add(testResult);
    }

    print("All result logs fetched.");
    print("Calling test.py to find statuses for each test.");
    print("");

    for (var testResult in testResults) {
      if (testResult is BuildBucketTestResult) {
        printBuild(testResult.build);
      }
      var statusExpectations = new StatusExpectations(testResult);
      await statusExpectations.loadStatusFiles();
      List<TestExpectationResult> results =
          statusExpectations.getTestResultsWithExpectation();
      printFailingTestExpectationResults(results);
      print("");
    }
  }
}

/// Prints a test result.
void printFailingTestExpectationResults(List<TestExpectationResult> results) {
  List<TestExpectationResult> failing =
      results.where((x) => !x.isSuccess()).toList();
  failing.sort((a, b) => a.result.name.compareTo(b.result.name));
  int index = 0;
  print("");
  failing.forEach((fail) => printFailingTest(fail, index++));
  if (index == 0) {
    print("\tNo failures found.");
  }
}

/// Prints a builder to stdout.
void printBuild(BuildBucketBuild build) {
  new ExtendedPrinter()
    ..println("${build.builder}")
    ..printLinePattern("=");
}

/// Prints a failing test to stdout.
void printFailingTest(TestExpectationResult result, int index) {
  var conf = result.configuration
      .toArgs(includeSelectors: false)
      .map((arg) => arg.replaceAll("--", ""));

  var extPrint = new ExtendedPrinter();
  if (index > 0) {
    extPrint.printLinePattern("*");
    extPrint.println("");
  }
  extPrint
    ..println("FAILED: ${getQualifiedNameForTest(result.result.name)}")
    ..printLinePattern("-")
    ..println("Result: ${result.result.result}")
    ..println("Expected: ${result.expectations()}");
  printStatusEntries(result.entries, extPrint);
  extPrint
    ..println("Configuration: ${conf.join(', ')}")
    ..println("")
    ..println(
        "To run locally (if you have the right architecture and runtime):")
    ..println(getReproductionCommand(result.configuration, result.result.name))
    ..println("");
}

void printStatusEntries(
    List<StatusSectionEntry> entries, ExtendedPrinter printer) {
  var oldPreceding = printer.preceding;
  printer.preceding = "  ";
  for (StatusSectionEntry entry in entries) {
    printer.println("${entry.statusFile.path}");
    printer.println("  [ ${entry.section.condition} ]");
    printer.println("    line ${entry.entry.lineNumber}: ${entry.entry.path} : "
        "${entry.entry.expectations}");
  }
  printer.preceding = oldPreceding;
}