pkg/frontend_server/test/frontend_server_flutter_suite.dart

// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async' show StreamSubscription, Timer;
import 'dart:convert' show jsonEncode;
import 'dart:io' show File, exitCode;
import 'dart:isolate' show Isolate, ReceivePort, SendPort;

import 'package:args/args.dart' show ArgParser;
import 'package:args/src/arg_results.dart';

import "frontend_server_flutter.dart" show Logger, compileTests;

const String suiteNamePrefix = "flutter_frontend";

class Options {
  final String configurationName;
  final bool verbose;
  final bool printFailureLog;
  final Uri outputDirectory;
  final String? testFilter;
  final String flutterDir;
  final String flutterPlatformDir;

  new(
    this.configurationName,
    this.verbose,
    this.printFailureLog,
    this.outputDirectory,
    this.testFilter,
    this.flutterDir,
    this.flutterPlatformDir,
  );

  static Options parse(List<String> args) {
    ArgParser parser = new ArgParser()
      ..addOption(
        "named-configuration",
        abbr: "n",
        defaultsTo: suiteNamePrefix,
        help: "configuration name to use for emitting json result files",
      )
      ..addOption(
        "output-directory",
        help: "directory to which results.json and logs.json are written",
      )
      ..addFlag(
        "verbose",
        abbr: "v",
        help: "print additional information",
        defaultsTo: false,
      )
      ..addFlag(
        "print",
        abbr: "p",
        help: "print failure logs",
        defaultsTo: false,
      )
      ..addOption("flutterDir")
      ..addOption("flutterPlatformDir");
    ArgResults parsedArguments = parser.parse(args);
    String outputPath = parsedArguments["output-directory"] ?? ".";
    Uri outputDirectory = Uri.base.resolveUri(new Uri.directory(outputPath));
    String? filter;
    if (parsedArguments.rest.length == 1) {
      filter = parsedArguments.rest.single;
      if (filter.startsWith("$suiteNamePrefix/")) {
        filter = filter.substring(suiteNamePrefix.length + 1);
      }
    }
    return new Options(
      parsedArguments["named-configuration"],
      parsedArguments["verbose"],
      parsedArguments["print"],
      outputDirectory,
      filter,
      parsedArguments["flutterDir"],
      parsedArguments["flutterPlatformDir"],
    );
  }
}

class ResultLogger extends Logger {
  final SuiteConfiguration suiteConfiguration;
  final Map<String, Stopwatch> stopwatches = {};
  final List<String> _log = <String>[];

  new(this.suiteConfiguration);

  void handleTestResult(String testName, bool matchedExpectations) {
    String fullTestName = "$suiteNamePrefix/$testName";
    suiteConfiguration.resultsPort.send(
      jsonEncode({
        "name": fullTestName,
        "configuration": suiteConfiguration.configurationName,
        "suite": suiteNamePrefix,
        "test_name": testName,
        "time_ms": stopwatches[testName]!.elapsedMilliseconds,
        "expected": "Pass",
        "result": matchedExpectations ? "Pass" : "Fail",
        "matches": matchedExpectations,
      }),
    );
    if (!matchedExpectations) {
      String failureLog = _log.join("\n");
      failureLog =
          "$failureLog\n\nRe-run this test: dart --enable-asserts "
          "pkg/frontend_server/test/test.dart "
          "--flutterDir=${suiteConfiguration.flutterDir} "
          "--flutterPlatformDir=${suiteConfiguration.flutterPlatformDir} "
          "-p $testName";
      suiteConfiguration.logsPort.send(
        jsonEncode({
          "name": fullTestName,
          "configuration": suiteConfiguration.configurationName,
          "result": matchedExpectations ? "OK" : "Failure",
          "log": failureLog,
        }),
      );
      if (suiteConfiguration.printFailureLog) {
        print('FAILED: $fullTestName');
        print(failureLog);
      }
    }
    if (suiteConfiguration.verbose) {
      String result = matchedExpectations ? "PASS" : "FAIL";
      print("$fullTestName: $result");
    }
  }

  @override
  void logTestStart(String testName) {
    stopwatches[testName] = new Stopwatch()..start();
    _log.clear();
  }

  @override
  void log(String s) {
    _log.add(s);
  }

  @override
  void notice(String s) {
    // ignored.
  }

  @override
  void logExpectedResult(String testName) {
    handleTestResult(testName, true);
  }

  @override
  void logUnexpectedResult(String testName) {
    handleTestResult(testName, false);
  }
}

const Duration timeoutDuration = const Duration(minutes: 45);

class SuiteConfiguration {
  final SendPort resultsPort;
  final SendPort logsPort;
  final bool verbose;
  final bool printFailureLog;
  final String configurationName;
  final String? testFilter;

  final int shard;
  final int shards;

  final String flutterDir;
  final String flutterPlatformDir;

  const new(
    this.resultsPort,
    this.logsPort,
    this.verbose,
    this.printFailureLog,
    this.configurationName,
    this.testFilter,
    this.flutterDir,
    this.flutterPlatformDir,
    this.shard,
    this.shards,
  );
}

Future<void> runSuite(SuiteConfiguration configuration) async {
  ResultLogger logger = new ResultLogger(configuration);
  try {
    await compileTests(
      configuration.flutterDir,
      configuration.flutterPlatformDir,
      logger,
      filter: configuration.testFilter,
      shard: configuration.shard,
      shards: configuration.shards,
    );
  } catch (e) {
    logger.logUnexpectedResult("startup");
  }
}

void writeLinesToFile(Uri uri, List<String> lines) {
  new File.fromUri(uri)
      .writeAsStringSync(lines.map((line) => "$line\n").join());
}

Future<void> main([List<String> arguments = const <String>[]]) async {
  List<String> results = [];
  List<String> logs = [];
  Options options = Options.parse(arguments);
  ReceivePort resultsPort = new ReceivePort()
    ..listen((resultEntry) => results.add(resultEntry));
  ReceivePort logsPort = new ReceivePort()
    ..listen((logEntry) => logs.add(logEntry));
  String? filter = options.testFilter;

  const int shards = 4;
  List<Future<bool>> futures = [];
  for (int shard = 0; shard < shards; shard++) {
    // Start the test suite in a new isolate.
    ReceivePort exitPort = new ReceivePort();
    ReceivePort errorPort = new ReceivePort();
    SuiteConfiguration configuration = new SuiteConfiguration(
      resultsPort.sendPort,
      logsPort.sendPort,
      options.verbose,
      options.printFailureLog,
      options.configurationName,
      filter,
      options.flutterDir,
      options.flutterPlatformDir,
      shard,
      shards,
    );
    Future<bool> future = new Future<bool>(() async {
      Stopwatch stopwatch = new Stopwatch()..start();
      print("Running suite shard $shard of $shards");
      Isolate isolate = await Isolate.spawn<SuiteConfiguration>(
        runSuite,
        configuration,
        onExit: exitPort.sendPort,
        onError: errorPort.sendPort,
      );
      bool gotError = false;
      StreamSubscription errorSubscription = errorPort.listen((message) {
        print("Got error: $message!");
        gotError = true;
        logs.add("$message");
      });
      bool timedOut = false;
      Timer timer = new Timer(timeoutDuration, () {
        timedOut = true;
        print(
          "Suite timed out after "
          "${timeoutDuration.inMilliseconds}ms",
        );
        isolate.kill(priority: Isolate.immediate);
      });
      await exitPort.first;
      await errorSubscription.cancel();
      timer.cancel();
      if (!timedOut && !gotError) {
        int seconds = stopwatch.elapsedMilliseconds ~/ 1000;
        print("Suite finished (shard #$shard) (took $seconds seconds)");
      }
      return timedOut || gotError;
    });
    futures.add(future);
  }

  // Wait for isolate to terminate and clean up.
  Iterable<bool> timeoutsOrCrashes = await Future.wait(futures);
  bool timeoutOrCrash = timeoutsOrCrashes.any((timeout) => timeout);
  resultsPort.close();
  logsPort.close();

  // Write results.json and logs.json.
  Uri resultJsonUri = options.outputDirectory.resolve("results.json");
  Uri logsJsonUri = options.outputDirectory.resolve("logs.json");
  writeLinesToFile(resultJsonUri, results);
  writeLinesToFile(logsJsonUri, logs);
  print(
    "Log files written to ${resultJsonUri.toFilePath()} and"
    " ${logsJsonUri.toFilePath()}",
  );

  exitCode = timeoutOrCrash ? 1 : 0;
}