dev/bots/utils.dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:core' hide print;
import 'dart:io' as system show exit;
import 'dart:io' hide exit;
import 'dart:math' as math;

import 'package:analyzer/dart/analysis/results.dart';
import 'package:analyzer/dart/ast/ast.dart';
import 'package:file/file.dart' as fs;
import 'package:file/local.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;

import 'run_command.dart';
import 'tool_subsharding.dart';

typedef ShardRunner = Future<void> Function();

/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
typedef OutputChecker = String? Function(CommandResult);

const Duration _quietTimeout = Duration(minutes: 10); // how long the output should be hidden between calls to printProgress before just being verbose

// If running from LUCI set to False.
final bool isLuci =  Platform.environment['LUCI_CI'] == 'True';
final bool hasColor = stdout.supportsAnsiEscapes && !isLuci;
final bool _isRandomizationOff = bool.tryParse(Platform.environment['TEST_RANDOMIZATION_OFF'] ?? '') ?? false;

final String bold = hasColor ? '\x1B[1m' : ''; // shard titles
final String red = hasColor ? '\x1B[31m' : ''; // errors
final String green = hasColor ? '\x1B[32m' : ''; // section titles, commands
final String yellow = hasColor ? '\x1B[33m' : ''; // indications that a test was skipped (usually renders orange or brown)
final String cyan = hasColor ? '\x1B[36m' : ''; // paths
final String reverse = hasColor ? '\x1B[7m' : ''; // clocks
final String gray = hasColor ? '\x1B[30m' : ''; // subtle decorative items (usually renders as dark gray)
final String white = hasColor ? '\x1B[37m' : ''; // last log line (usually renders as light gray)
final String reset = hasColor ? '\x1B[0m' : '';

final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : '';
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
final String pubCache = path.join(flutterRoot, '.pub-cache');
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
final String luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
final bool runningInDartHHHBot =
    luciBotId.startsWith('luci-dart-') || luciBotId.startsWith('dart-tests-');

const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';
const String kTestHarnessShardName = 'test_harness_tests';
const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';

/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};

/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
final List<String> flutterTestArgs = <String>[];


const int kESC = 0x1B;
const int kOpenSquareBracket = 0x5B;
const int kCSIParameterRangeStart = 0x30;
const int kCSIParameterRangeEnd = 0x3F;
const int kCSIIntermediateRangeStart = 0x20;
const int kCSIIntermediateRangeEnd = 0x2F;
const int kCSIFinalRangeStart = 0x40;
const int kCSIFinalRangeEnd = 0x7E;

String get redLine {
  if (hasColor) {
    return '$red${'━' * stdout.terminalColumns}$reset';
  }
  return '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━';
}

String get clock {
  final DateTime now = DateTime.now();
  return '$reverse▌'
         '${now.hour.toString().padLeft(2, "0")}:'
         '${now.minute.toString().padLeft(2, "0")}:'
         '${now.second.toString().padLeft(2, "0")}'
         '▐$reset';
}

String prettyPrintDuration(Duration duration) {
  String result = '';
  final int minutes = duration.inMinutes;
  if (minutes > 0) {
    result += '${minutes}min ';
  }
  final int seconds = duration.inSeconds - minutes * 60;
  final int milliseconds = duration.inMilliseconds - (seconds * 1000 + minutes * 60 * 1000);
  result += '$seconds.${milliseconds.toString().padLeft(3, "0")}s';
  return result;
}

typedef PrintCallback = void Function(Object? line);
typedef VoidCallback = void Function();

// Allow print() to be overridden, for tests.
//
// Files that import this library should not import `print` from dart:core
// and should not use dart:io's `stdout` or `stderr`.
//
// By default this hides log lines between `printProgress` calls unless a
// timeout expires or anything calls `foundError`.
//
// Also used to implement `--verbose` in test.dart.
PrintCallback print = _printQuietly;

// Called by foundError and used to implement `--abort-on-error` in test.dart.
VoidCallback? onError;

bool get hasError => _hasError;
bool _hasError = false;

List<List<String>> _errorMessages = <List<String>>[];

final List<String> _pendingLogs = <String>[];
Timer? _hideTimer; // When this is null, the output is verbose.

void foundError(List<String> messages) {
  assert(messages.isNotEmpty);
  // Make the error message easy to notice in the logs by
  // wrapping it in a red box.
  final int width = math.max(15, (hasColor ? stdout.terminalColumns : 80) - 1);
  final String title = 'ERROR #${_errorMessages.length + 1}';
  print('$red╔═╡$bold$title$reset$red╞═${"═" * (width - 4 - title.length)}');
  for (final String message in messages.expand((String line) => line.split('\n'))) {
    print('$red║$reset $message');
  }
  print('$red╚${"═" * width}');
  // Normally, "print" actually prints to the log. To make the errors visible,
  // and to include useful context, print the entire log up to this point, and
  // clear it. Subsequent messages will continue to not be logged until there is
  // another error.
  _pendingLogs.forEach(_printLoudly);
  _pendingLogs.clear();
  _errorMessages.add(messages);
  _hasError = true;
  onError?.call();
}

@visibleForTesting
void resetErrorStatus() {
  _hasError = false;
  _errorMessages.clear();
  _pendingLogs.clear();
  _hideTimer?.cancel();
  _hideTimer = null;
}

Never reportSuccessAndExit(String message) {
  _hideTimer?.cancel();
  _hideTimer = null;
  print('$clock $message$reset');
  system.exit(0);
}

Never reportErrorsAndExit(String message) {
  _hideTimer?.cancel();
  _hideTimer = null;
  print('$clock $message$reset');
  print(redLine);
  print('${red}For your convenience, the error messages reported above are repeated here:$reset');
  final bool printSeparators = _errorMessages.any((List<String> messages) => messages.length > 1);
  if (printSeparators) {
    print('  🙙  🙛  ');
  }
  for (int index = 0; index < _errorMessages.length * 2 - 1; index += 1) {
    if (index.isEven) {
      _errorMessages[index ~/ 2].forEach(print);
    } else if (printSeparators) {
      print('  🙙  🙛  ');
    }
  }
  print(redLine);
  print('You may find the errors by searching for "╡ERROR #" in the logs.');
  system.exit(1);
}

void printProgress(String message) {
  _pendingLogs.clear();
  _hideTimer?.cancel();
  _hideTimer = null;
  print('$clock $message$reset');
  if (hasColor) {
    // This sets up a timer to switch to verbose mode when the tests take too long,
    // so that if a test hangs we can see the logs.
    // (This is only supported with a color terminal. When the terminal doesn't
    // support colors, the scripts just print everything verbosely, that way in
    // CI there's nothing hidden.)
    _hideTimer = Timer(_quietTimeout, () {
      _hideTimer = null;
      _pendingLogs.forEach(_printLoudly);
      _pendingLogs.clear();
    });
  }
}

final Pattern _lineBreak = RegExp(r'[\r\n]');

void _printQuietly(Object? message) {
  // The point of this function is to avoid printing its output unless the timer
  // has gone off in which case the function assumes verbose mode is active and
  // prints everything. To show that progress is still happening though, rather
  // than showing nothing at all, it instead shows the last line of output and
  // keeps overwriting it. To do this in color mode, carefully measures the line
  // of text ignoring color codes, which is what the parser below does.
  if (_hideTimer != null) {
    _pendingLogs.add(message.toString());
    String line = '$message'.trimRight();
    final int start = line.lastIndexOf(_lineBreak) + 1;
    int index = start;
    int length = 0;
    while (index < line.length && length < stdout.terminalColumns) {
      if (line.codeUnitAt(index) == kESC) { // 0x1B
        index += 1;
        if (index < line.length && line.codeUnitAt(index) == kOpenSquareBracket) { // 0x5B, [
          // That was the start of a CSI sequence.
          index += 1;
          while (index < line.length && line.codeUnitAt(index) >= kCSIParameterRangeStart
                                     && line.codeUnitAt(index) <= kCSIParameterRangeEnd) { // 0x30..0x3F
            index += 1; // ...parameter bytes...
          }
          while (index < line.length && line.codeUnitAt(index) >= kCSIIntermediateRangeStart
                                     && line.codeUnitAt(index) <= kCSIIntermediateRangeEnd) { // 0x20..0x2F
            index += 1; // ...intermediate bytes...
          }
          if (index < line.length && line.codeUnitAt(index) >= kCSIFinalRangeStart
                                  && line.codeUnitAt(index) <= kCSIFinalRangeEnd) { // 0x40..0x7E
            index += 1; // ...final byte.
          }
        }
      } else {
        index += 1;
        length += 1;
      }
    }
    line = line.substring(start, index);
    if (line.isNotEmpty) {
      stdout.write('\r\x1B[2K$white$line$reset');
    }
  } else {
    _printLoudly('$message');
  }
}

void _printLoudly(String message) {
  if (hasColor) {
    // Overwrite the last line written by _printQuietly.
    stdout.writeln('\r\x1B[2K$reset${message.trimRight()}');
  } else {
    stdout.writeln(message);
  }
}

// THE FOLLOWING CODE IS A VIOLATION OF OUR STYLE GUIDE
// BECAUSE IT INTRODUCES A VERY FLAKY RACE CONDITION
// https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#never-check-if-a-port-is-available-before-using-it-never-add-timeouts-and-other-race-conditions
// DO NOT USE THE FOLLOWING FUNCTIONS
// DO NOT WRITE CODE LIKE THE FOLLOWING FUNCTIONS
// https://github.com/flutter/flutter/issues/109474

int _portCounter = 8080;

/// Finds the next available local port.
Future<int> findAvailablePortAndPossiblyCauseFlakyTests() async {
  while (!await _isPortAvailable(_portCounter)) {
    _portCounter += 1;
  }
  return _portCounter++;
}

Future<bool> _isPortAvailable(int port) async {
  try {
    final RawSocket socket = await RawSocket.connect('localhost', port);
    socket.shutdown(SocketDirection.both);
    await socket.close();
    return false;
  } on SocketException {
    return true;
  }
}

String locationInFile(ResolvedUnitResult unit, AstNode node, String workingDirectory) {
  return '${path.relative(path.relative(unit.path, from: workingDirectory))}:${unit.lineInfo.getLocation(node.offset).lineNumber}';
}

// The seed used to shuffle tests. If not passed with
// --test-randomize-ordering-seed=<seed> on the command line, it will be set the
// first time it is accessed. Pass zero to turn off shuffling.
String? _shuffleSeed;

set shuffleSeed(String? newSeed) {
  _shuffleSeed = newSeed;
}

String get shuffleSeed {
  if (_shuffleSeed != null) {
    return _shuffleSeed!;
  }
  // Attempt to load from the command-line argument
  final String? seedArg = Platform.environment['--test-randomize-ordering-seed'];
  if (seedArg != null) {
    return seedArg;
  }
  // Fallback to the original time-based seed generation
  final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7));
  _shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}';
  return _shuffleSeed!;
}

// TODO(sigmund): includeLocalEngineEnv should default to true. Currently we
// only enable it on flutter-web test because some test suites do not work
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
Future<void> runDartTest(String workingDirectory, {
  List<String>? testPaths,
  bool enableFlutterToolAsserts = true,
  bool useBuildRunner = false,
  String? coverage,
  bool forceSingleCore = false,
  Duration? perTestTimeout,
  bool includeLocalEngineEnv = false,
  bool ensurePrecompiledTool = true,
  bool shuffleTests = true,
  bool collectMetrics = false,
}) async {
  int? cpus;
  final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
  if (cpuVariable != null) {
    cpus = int.tryParse(cpuVariable, radix: 10);
    if (cpus == null) {
      foundError(<String>[
        '${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset',
        'Actual value: "$cpuVariable"',
      ]);
      return;
    }
  } else {
    cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
  }
  // Integration tests that depend on external processes like chrome
  // can get stuck if there are multiple instances running at once.
  if (forceSingleCore) {
    cpus = 1;
  }

  const LocalFileSystem fileSystem = LocalFileSystem();
  final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
  final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
  final List<String> args = <String>[
    'run',
    'test',
    '--reporter=expanded',
    '--file-reporter=json:${metricFile.path}',
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
    '-j$cpus',
    if (!hasColor)
      '--no-color',
    if (coverage != null)
      '--coverage=$coverage',
    if (perTestTimeout != null)
      '--timeout=${perTestTimeout.inMilliseconds}ms',
    if (testPaths != null)
      for (final String testPath in testPaths)
        testPath,
  ];
  final Map<String, String> environment = <String, String>{
    'FLUTTER_ROOT': flutterRoot,
    if (includeLocalEngineEnv)
      ...localEngineEnv,
    if (Directory(pubCache).existsSync())
      'PUB_CACHE': pubCache,
  };
  if (enableFlutterToolAsserts) {
    adjustEnvironmentToEnableFlutterAsserts(environment);
  }
  if (ensurePrecompiledTool) {
    // We rerun the `flutter` tool here just to make sure that it is compiled
    // before tests run, because the tests might time out if they have to rebuild
    // the tool themselves.
    await runCommand(flutter, <String>['--version'], environment: environment);
  }
  await runCommand(
    dart,
    args,
    workingDirectory: workingDirectory,
    environment: environment,
    removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
  );

  final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
  final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
  info.writeAsStringSync(json.encode(test.errors));

  if (collectMetrics) {
    try {
      final List<String> testList = <String>[];
      final Map<int, TestSpecs> allTestSpecs = test.allTestSpecs;
      for (final TestSpecs testSpecs in allTestSpecs.values) {
        testList.add(testSpecs.toJson());
      }
      if (testList.isNotEmpty) {
        final String testJson = json.encode(testList);
        final File testResults = fileSystem.file(
            path.join(flutterRoot, 'test_results.json'));
        testResults.writeAsStringSync(testJson);
      }
    } on fs.FileSystemException catch (e) {
      print('Failed to generate metrics: $e');
    }
  }

  // metriciFile is a transitional file that needs to be deleted once it is parsed.
  // TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
  // https://github.com/flutter/flutter/issues/146003
  metricFile.deleteSync();
}

Future<void> runFlutterTest(String workingDirectory, {
  String? script,
  bool expectFailure = false,
  bool printOutput = true,
  OutputChecker? outputChecker,
  List<String> options = const <String>[],
  Map<String, String>? environment,
  List<String> tests = const <String>[],
  bool shuffleTests = true,
  bool fatalWarnings = true,
}) async {
  assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');

  final List<String> tags = <String>[];
  // Recipe-configured reduced test shards will only execute tests with the
  // appropriate tag.
  if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
    tags.addAll(<String>['-t', 'reduced-test-set']);
  }

  const LocalFileSystem fileSystem = LocalFileSystem();
  final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
  final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
  final List<String> args = <String>[
    'test',
    '--reporter=expanded',
    '--file-reporter=json:${metricFile.path}',
    if (shuffleTests && !_isRandomizationOff) '--test-randomize-ordering-seed=$shuffleSeed',
    if (fatalWarnings) '--fatal-warnings',
    ...options,
    ...tags,
    ...flutterTestArgs,
  ];

  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
      foundError(<String>[
        '${red}Could not find test$reset: $green$fullScriptPath$reset',
        'Working directory: $cyan$workingDirectory$reset',
        'Script: $green$script$reset',
        if (!printOutput)
          'This is one of the tests that does not normally print output.',
      ]);
      return;
    }
    args.add(script);
  }

  args.addAll(tests);

  final OutputMode outputMode = outputChecker == null && printOutput
    ? OutputMode.print
    : OutputMode.capture;

  final CommandResult result = await runCommand(
    flutter,
    args,
    workingDirectory: workingDirectory,
    expectNonZeroExit: expectFailure,
    outputMode: outputMode,
    environment: environment,
  );

  // metriciFile is a transitional file that needs to be deleted once it is parsed.
  // TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
  // https://github.com/flutter/flutter/issues/146003
  metricFile.deleteSync();

  if (outputChecker != null) {
    final String? message = outputChecker(result);
    if (message != null) {
      foundError(<String>[message]);
    }
  }
}

/// This will force the next run of the Flutter tool (if it uses the provided
/// environment) to have asserts enabled, by setting an environment variable.
void adjustEnvironmentToEnableFlutterAsserts(Map<String, String> environment) {
  // If an existing env variable exists append to it, but only if
  // it doesn't appear to already include enable-asserts.
  String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
  if (!toolsArgs.contains('--enable-asserts')) {
    toolsArgs += ' --enable-asserts';
  }
  environment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
}

Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);

Future<void> runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async {
  final List<ShardRunner> sublist = selectIndexOfTotalSubshard<ShardRunner>(tests);
  for (final ShardRunner test in sublist) {
    await test();
  }
}
/// Parse (one-)index/total-named subshards from environment variable SUBSHARD
/// and equally distribute [tests] between them.
/// Subshard format is "{index}_{total number of shards}".
/// The scheduler can change the number of total shards without needing an additional
/// commit in this repository.
///
/// Examples:
/// 1_3
/// 2_3
/// 3_3
List<T> selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubshardKey}) {
  // Example: "1_3" means the first (one-indexed) shard of three total shards.
  final String? subshardName = Platform.environment[subshardKey];
  if (subshardName == null) {
    print('$kSubshardKey environment variable is missing, skipping sharding');
    return tests;
  }
  printProgress('$bold$subshardKey=$subshardName$reset');

  final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
  final Match? match = pattern.firstMatch(subshardName);
  if (match == null || match.groupCount != 2) {
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Expected format "[int]_[int]" ex. "1_3"',
    ]);
    throw Exception('Invalid subshard name: $subshardName');
  }
  // One-indexed.
  final int index = int.parse(match.group(1)!);
  final int total = int.parse(match.group(2)!);
  if (index > total) {
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Index number must be greater or equal to total.',
    ]);
    return <T>[];
  }

  final int testsPerShard = (tests.length / total).ceil();
  final int start = (index - 1) * testsPerShard;
  final int end = math.min(index * testsPerShard, tests.length);

  print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
  return tests.sublist(start, end);
}

Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
  String? item = Platform.environment[key];
  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
    assert(positionInTaskName < parts.length);
    item = parts[positionInTaskName];
  }
  if (item == null) {
    for (final String currentItem in items.keys) {
      printProgress('$bold$key=$currentItem$reset');
      await items[currentItem]!();
    }
  } else {
    printProgress('$bold$key=$item$reset');
    if (!items.containsKey(item)) {
      foundError(<String>[
        '${red}Invalid $name: $item$reset',
        'The available ${name}s are: ${items.keys.join(", ")}',
      ]);
      return;
    }
    await items[item]!();
  }
}

/// Checks the given file's contents to determine if they match the allowed
/// pattern for version strings.
///
/// Returns null if the contents are good. Returns a string if they are bad.
/// The string is an error message.
Future<String?> verifyVersion(File file) async {
  final RegExp pattern = RegExp(
    r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$');
  if (!file.existsSync()) {
    return 'The version logic failed to create the Flutter version file.';
  }
  final String version = await file.readAsString();
  if (version == '0.0.0-unknown') {
    return 'The version logic failed to determine the Flutter version.';
  }
  if (!version.contains(pattern)) {
    return 'The version logic generated an invalid version string: "$version".';
  }
  return null;
}