tools/gardening/lib/src/compare_failures_impl.dart

// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// Compares the test log of a build step with previous builds.
///
/// Use this to detect flakiness of failures, especially timeouts.

import 'dart:async';

import 'bot.dart';
import 'buildbot_structures.dart';
import 'buildbot_data.dart';
import 'cache_new.dart';
import 'logger.dart';
import 'luci_api.dart' hide Timing;
import 'luci.dart';
import 'util.dart';

Future mainInternal(Bot bot, List<String> args,
    {int runCount: 10,
    String commit,
    bool verbose: false,
    bool noCache: false,
    bool forcePastResults: false}) async {
  printBuildResultsSummary(
      await loadBuildResults(bot, args,
          runCount: runCount,
          commit: commit,
          verbose: verbose,
          noCache: noCache,
          forcePastResults: forcePastResults),
      args);
}

RegExp logdogUrlRegexp = new RegExp(r'https://logs.chromium.org/.*client.dart');

/// Loads [BuildResult]s for the [runCount] last builds for the build(s) in
/// [args]. [args] can be a list of [BuildGroup] names or a list of log uris.
Future<Map<BuildUri, List<BuildResult>>> loadBuildResults(
    Bot bot, List<String> args,
    {int runCount: 10,
    String commit,
    bool verbose: false,
    bool noCache: false,
    bool forcePastResults: false}) async {
  List<BuildUri> buildUriList = <BuildUri>[];
  List<BuildDetail> buildDetails;
  if (commit != null) {
    LuciApi luci = new LuciApi();
    Logger logger = createLogger(verbose: verbose);
    CreateCacheFunction createCache =
        createCacheFunction(logger, disableCache: noCache);
    buildDetails = await fetchBuildsForCommmit(
        luci, logger, DART_CLIENT, commit, createCache, 25);
    if (buildDetails.isEmpty) {
      print('No builds found for $commit');
    } else if (verbose) {
      log('Found builds for commit $commit:');
      buildDetails.forEach((b) => log(' ${b.botName}: ${b.buildNumber}'));
    } else {
      print('Found ${buildDetails.length} builds for commit $commit.');
    }
  }

  BuildUri updateWithCommit(BuildUri buildUri) {
    if (buildDetails == null) return buildUri;
    for (BuildDetail buildDetail in buildDetails) {
      if (buildDetail.botName == buildUri.botName) {
        return buildUri.withBuildNumber(buildDetail.buildNumber);
      }
    }
    print('No build number for $commit found for $buildUri.');
    return buildUri;
  }

  for (BuildGroup buildGroup in buildGroups) {
    if (args.contains(buildGroup.groupName)) {
      buildUriList.addAll(buildGroup
          .createUris(bot.mostRecentBuildNumber)
          .map(updateWithCommit));
    } else {
      for (BuildSubgroup subGroup in buildGroup.subgroups) {
        for (String arg in args) {
          if (subGroup.shardNames.contains(arg)) {
            buildUriList.addAll(subGroup
                .createUris(bot.mostRecentBuildNumber)
                .map(updateWithCommit));
            // Break out to not include more from same group.
            break;
          }
        }
      }
    }
  }
  if (buildUriList.isEmpty) {
    for (String url in args) {
      try {
        buildUriList.add(updateWithCommit(new BuildUri.fromUrl(url)));
      } catch (e) {
        print("'$url' is not a valid build bot url: $e");
      }
    }
  }

  Map<BuildUri, List<BuildResult>> pastResultsMap =
      <BuildUri, List<BuildResult>>{};
  List<BuildResult> buildResults = await bot.readResults(buildUriList);
  if (buildResults.length != buildUriList.length) {
    print('Result mismatch: Pulled ${buildUriList.length} uris, '
        'received ${buildResults.length} results.');
  }
  for (int index = 0; index < buildResults.length; index++) {
    BuildUri buildUri = buildUriList[index];
    BuildResult buildResult = buildResults[index];
    List<BuildResult> results = await readPastResults(
        bot, buildUri, buildResult, runCount,
        forcePastResults: forcePastResults);
    pastResultsMap[buildUri] = results;
  }
  return pastResultsMap;
}

/// Prints summaries for the [buildResults].
void printBuildResultsSummary(
    Map<BuildUri, List<BuildResult>> buildResults, List<String> args) {
  List<Summary> emptySummaries = <Summary>[];
  List<Summary> nonEmptySummaries = <Summary>[];
  buildResults.forEach((BuildUri buildUri, List<BuildResult> results) {
    Summary summary = new Summary(buildUri, results);
    if (summary.isEmpty) {
      emptySummaries.add(summary);
    } else {
      nonEmptySummaries.add(summary);
    }
  });
  StringBuffer sb = new StringBuffer();
  if (nonEmptySummaries.isEmpty) {
    if (emptySummaries.isNotEmpty) {
      if (LOG || emptySummaries.length < 3) {
        if (emptySummaries.length == 1) {
          sb.writeln('No errors found for build bot:');
          sb.write(emptySummaries.single.name);
        } else {
          sb.writeln('No errors found for any of these build bots:');
          for (Summary summary in emptySummaries) {
            sb.writeln('${summary.name}');
          }
        }
      } else {
        sb.write('No errors found for any of the '
            '${emptySummaries.length} bots.');
      }
    } else {
      sb.write('No build bot results found for args: ${args}');
    }
  } else {
    for (Summary summary in nonEmptySummaries) {
      summary.printOn(sb);
    }
    if (emptySummaries.isNotEmpty) {
      if (LOG || emptySummaries.length < 3) {
        sb.writeln('No errors found for the remaining build bots:');
        for (Summary summary in emptySummaries) {
          sb.writeln('${summary.name}');
        }
      } else {
        sb.write(
            'No errors found for the ${emptySummaries.length} remaining bots.');
      }
    }
  }
  print(sb);
}

/// Creates a [BuildResult] for [buildUri] and, if it contains failures, the
/// [BuildResult]s for the previous [runCount] builds.
Future<List<BuildResult>> readPastResults(
    Bot bot, BuildUri buildUri, BuildResult summary, int runCount,
    {bool forcePastResults: false}) async {
  List<BuildResult> summaries = <BuildResult>[];
  if (summary == null) {
    print('No result found for $buildUri');
    return summaries;
  }
  summaries.add(summary);
  if (summary.hasFailures || forcePastResults) {
    summaries.addAll(await bot.readHistoricResults(summary.buildUri.prev(),
        previousCount: runCount - 1));
  }
  return summaries;
}

class Summary {
  final BuildUri buildUri;
  final List<BuildResult> results;
  final Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>();
  final Set<TestConfiguration> errorIds = new Set<TestConfiguration>();

  Summary(this.buildUri, this.results) {
    for (BuildResult result in results) {
      if (result == null) continue;
      timeoutIds
          .addAll(result.timeouts.map((TestFailure failure) => failure.id));
      errorIds.addAll(result.errors.map((TestFailure failure) => failure.id));
    }
  }

  bool get isEmpty => timeoutIds.isEmpty && errorIds.isEmpty;

  /// Generate a summary of the timeouts and other failures in [results].
  void printOn(StringBuffer sb) {
    if (timeoutIds.isNotEmpty) {
      Map<TestConfiguration, Map<int, Map<String, Timing>>> map =
          <TestConfiguration, Map<int, Map<String, Timing>>>{};
      Set<String> stepNames = new Set<String>();
      for (BuildResult result in results) {
        for (Timing timing in result.timings) {
          Map<int, Map<String, Timing>> builds = map.putIfAbsent(
              timing.step.id, () => <int, Map<String, Timing>>{});
          stepNames.add(timing.step.stepName);
          builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[
              timing.step.stepName] = timing;
        }
      }
      sb.write('Timeouts for ${name} :\n');
      map.forEach(
          (TestConfiguration id, Map<int, Map<String, Timing>> timings) {
        if (!timeoutIds.contains(id)) return;
        sb.write('$id\n');
        sb.write(
            '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n');
        for (BuildResult result in results) {
          int buildNumber = result.buildUri.buildNumber;
          Map<String, Timing> steps = timings[buildNumber] ?? const {};
          sb.write(padRight(' ${buildNumber}: ', 8));
          for (String stepName in stepNames) {
            Timing timing = steps[stepName];
            if (timing != null) {
              sb.write(' ${timing.time}');
            } else {
              sb.write(' --------------');
            }
          }
          sb.write('\n');
        }
        sb.write('\n');
      });
    }
    if (errorIds.isNotEmpty) {
      Map<TestConfiguration, Map<int, TestFailure>> map =
          <TestConfiguration, Map<int, TestFailure>>{};
      for (BuildResult result in results) {
        for (TestFailure failure in result.errors) {
          map.putIfAbsent(failure.id, () => <int, TestFailure>{})[
              failure.uri.buildNumber] = failure;
        }
      }
      sb.write('Errors for ${name} :\n');
      // TODO(johnniwinther): Improve comparison of non-timeouts.
      map.forEach((TestConfiguration id, Map<int, TestFailure> failures) {
        if (!errorIds.contains(id)) return;
        sb.write('$id\n');
        for (BuildResult result in results) {
          int buildNumber = result.buildUri.buildNumber;
          TestFailure failure = failures[buildNumber];
          sb.write(padRight(' ${buildNumber}: ', 8));
          if (failure != null) {
            sb.write(padRight(failure.expected, 10));
            sb.write(' / ');
            sb.write(padRight(failure.actual, 10));
          } else {
            sb.write(' ' * 10);
            sb.write(' / ');
            sb.write(padRight('-- OK --', 10));
          }
          sb.write('\n');
        }
        sb.write('\n');
      });
    }
    if (timeoutIds.isEmpty && errorIds.isEmpty) {
      sb.write('No errors found for ${name}');
    }
  }

  String get name => results.isNotEmpty
      // Use the first result as name since it most likely has an absolute build
      // number.
      ? results.first.buildUri.toString()
      : buildUri.toString();
}