pkgs/coverage/bin/test_with_coverage.dart

// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:args/args.dart';
import 'package:cli_util/cli_util.dart';
import 'package:coverage/src/coverage_options.dart';

import 'package:coverage/src/hitmap.dart';
import 'package:coverage/src/util.dart';
import 'package:meta/meta.dart';
import 'package:package_config/package_config.dart';
import 'package:path/path.dart' as path;

import 'collect_coverage.dart' as collect_coverage;
import 'format_coverage.dart' as format_coverage;

final _allProcesses = <Process>[];
String get _dartExecutable =>
    dartExecutable ?? (throw StateError('Could not locate a Dart executable.'));

const _supportedPlatforms = ['vm', 'chrome'];

Future<void> _dartRun(
  List<String> args, {
  void Function(String)? onStdout,
  void Function(String)? onStderr,
  String? workingDirectory,
}) async {
  final process = await Process.start(
    _dartExecutable,
    args,
    workingDirectory: workingDirectory,
  );
  _allProcesses.add(process);

  void listen(
    Stream<List<int>> stream,
    IOSink sink,
    void Function(String)? onLine,
  ) {
    final broadStream = stream.asBroadcastStream();
    broadStream.listen(sink.add);
    if (onLine != null) {
      broadStream.lines().listen(onLine);
    }
  }

  listen(process.stdout, stdout, onStdout);
  listen(process.stderr, stderr, onStderr);

  final result = await process.exitCode;
  if (result != 0) {
    throw ProcessException(_dartExecutable, args, '', result);
  }
}

void _killSubprocessesAndExit(ProcessSignal signal) {
  for (final process in _allProcesses) {
    process.kill(signal);
  }
  exit(1);
}

void _watchExitSignal(ProcessSignal signal) {
  signal.watch().listen(_killSubprocessesAndExit);
}

void _registerSignalHandlers() {
  _watchExitSignal(ProcessSignal.sighup);
  _watchExitSignal(ProcessSignal.sigint);
  if (!Platform.isWindows) {
    _watchExitSignal(ProcessSignal.sigterm);
  }
}

ArgParser _createArgParser(CoverageOptions defaultOptions) => ArgParser()
  ..addOption(
    'package',
    help: 'Root directory of the package to test.',
    defaultsTo: defaultOptions.packageDirectory,
  )
  ..addOption(
    'package-name',
    help:
        'Name of the package to test. '
        'Deduced from --package if not provided. '
        'DEPRECATED: use --scope-output',
  )
  ..addOption('port', help: 'VM service port. Defaults to using any free port.')
  ..addOption(
    'out',
    defaultsTo: defaultOptions.outputDirectory,
    abbr: 'o',
    help: 'Output directory. Defaults to <package-dir>/coverage.',
  )
  ..addOption(
    'test',
    help: 'Test script or directory to run.',
    defaultsTo: defaultOptions.testScript,
  )
  ..addFlag(
    'function-coverage',
    abbr: 'f',
    defaultsTo: defaultOptions.functionCoverage,
    help: 'Collect function coverage info.',
  )
  ..addFlag(
    'branch-coverage',
    abbr: 'b',
    defaultsTo: defaultOptions.branchCoverage,
    help: 'Collect branch coverage info.',
  )
  ..addOption(
    'fail-under',
    help: 'Fail if coverage is less than the given percentage (0-100)',
  )
  ..addMultiOption(
    'scope-output',
    defaultsTo: defaultOptions.scopeOutput,
    help:
        'restrict coverage results so that only scripts that start with '
        'the provided package path are considered. Defaults to the name of '
        'the current package (including all subpackages, if this is a '
        'workspace).',
  )
  ..addOption(
    'platform',
    abbr: 'p',
    allowed: _supportedPlatforms,
    help:
        'The platform on which to run the tests. '
        'Supported: ${_supportedPlatforms.join(', ')}. '
        'Web coverage is measured on compiled JavaScript, so its line counts '
        'and percentages are not comparable to vm: code the compiler drops '
        'cannot be reported as uncovered. Re-tune --fail-under per platform.',
  )
  ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help.');

class Flags {
  Flags(
    this.packageDir,
    this.outDir,
    this.port,
    this.testScript,
    this.functionCoverage,
    this.branchCoverage,
    this.scopeOutput,
    this.failUnder,
    this.platform, {
    required this.rest,
  });

  final String packageDir;
  final String outDir;
  final String port;
  final String testScript;
  final bool functionCoverage;
  final bool branchCoverage;
  final List<String> scopeOutput;
  final String? failUnder;
  final String? platform;
  final List<String> rest;
}

@visibleForTesting
Future<Flags> parseArgs(
  List<String> arguments,
  CoverageOptions defaultOptions,
) async {
  final parser = _createArgParser(defaultOptions);

  void printUsage() {
    print('''
Runs tests and collects coverage for a package.

By default this script assumes it's being run from the root directory of a
package, and outputs a coverage.json and lcov.info to ./coverage/

Usage: test_with_coverage [OPTIONS...] [-- <test script OPTIONS>]

${parser.usage}
''');
  }

  Never fail(String msg) {
    print('\n$msg\n');
    printUsage();
    exit(1);
  }

  final ArgResults args;
  try {
    args = parser.parse(arguments);
  } on ArgParserException catch (e) {
    fail(e.message);
  }

  if (args['help'] as bool) {
    printUsage();
    exit(0);
  }

  final packageDir = path.normalize(path.absolute(args['package'] as String));
  if (!FileSystemEntity.isDirectorySync(packageDir)) {
    fail('--package is not a valid directory.');
  }

  final pubspecPath = getPubspecPath(packageDir);
  if (!File(pubspecPath).existsSync()) {
    fail(
      "Couldn't find $pubspecPath. Make sure this command is run in a "
      'package directory, or pass --package to explicitly set the directory.',
    );
  }

  final platform = args.option('platform');
  if (platform == 'chrome') {
    if (args.flag('function-coverage')) {
      fail('--function-coverage is not supported for web platforms.');
    }
    if (args.flag('branch-coverage')) {
      fail('--branch-coverage is not supported for web platforms.');
    }
  }

  // Extra args are forwarded to `dart test`, where a second platform flag
  // would silently override the validated one.
  for (final arg in args.rest) {
    if (arg == '-p' ||
        arg == '--platform' ||
        arg.startsWith('-p=') ||
        arg.startsWith('--platform=')) {
      fail(
        'Pass the platform with --platform, not as an argument to the test '
        'script. Coverage can only be collected for the platform this '
        'command runs.',
      );
    }
  }

  return Flags(
    packageDir,
    args.option('out') ?? path.join(packageDir, 'coverage'),
    args.option('port') ?? '0',
    args.option('test')!,
    args.flag('function-coverage'),
    args.flag('branch-coverage'),
    args.multiOption('scope-output'),
    args.option('fail-under'),
    platform,
    rest: args.rest,
  );
}

Future<void> _validatePackageConfig(String packageDir) async {
  final pkgConfig = await findPackageConfig(Directory(packageDir));
  if (pkgConfig == null) {
    stderr.writeln(
      'warning: package_config.json was not found for $packageDir. '
      'Make sure to run "dart pub get" in the package directory.',
    );
    return;
  }
  final testPkg = pkgConfig['test'] ?? pkgConfig['test_core'];
  if (testPkg == null) {
    stderr.writeln(
      'warning: package:test is not listed in package_config.json for '
      '$packageDir. Make sure to run "dart pub get" in the package '
      'directory.',
    );
  }
}

Future<int> _runVmTestsAndCollectCoverage(Flags flags, String outJson) async {
  final serviceUriCompleter = Completer<Uri>();
  final testArgs = [
    if (flags.branchCoverage) '--branch-coverage',
    'run',
    '--pause-isolates-on-exit',
    '--disable-service-auth-codes',
    '--enable-vm-service=${flags.port}',
    flags.testScript,
    ...flags.rest,
  ];
  final testProcess = _dartRun(
    testArgs,
    workingDirectory: flags.packageDir,
    onStdout: (line) {
      if (!serviceUriCompleter.isCompleted) {
        final uri = extractVMServiceUri(line);
        if (uri != null) {
          serviceUriCompleter.complete(uri);
        }
      }
    },
    onStderr: (line) {
      if (!serviceUriCompleter.isCompleted &&
          line.contains('Could not start the VM service')) {
        _killSubprocessesAndExit(ProcessSignal.sigkill);
      }
    },
  );

  var testExitCode = 0;
  final testProcessDone = testProcess
      .then((_) => 0)
      .catchError(
        (Object error) => testExitCode = (error as ProcessException).errorCode,
        test: (error) => error is ProcessException,
      );

  // If the test process exits before reporting a VM service URI there is
  // nothing to collect from, and waiting for the URI would hang forever.
  unawaited(
    testProcessDone.then((code) {
      if (!serviceUriCompleter.isCompleted) {
        stderr.writeln(
          'Test process exited before the VM service was ready '
          '(exit code $code).',
        );
        exit(code == 0 ? 1 : code);
      }
    }),
  );

  final serviceUri = await serviceUriCompleter.future;

  final scopes = flags.scopeOutput.isEmpty
      ? getAllWorkspaceNames(flags.packageDir)
      : flags.scopeOutput;
  final collection = collect_coverage.main([
    '--wait-paused',
    '--resume-isolates',
    '--uri=$serviceUri',
    for (final scope in scopes) '--scope-output=$scope',
    if (flags.branchCoverage) '--branch-coverage',
    if (flags.functionCoverage) '--function-coverage',
    '-o',
    outJson,
  ]);

  // The test process is paused on exit, so collection is what lets it
  // finish. If it exits with an error first, it never reached that pause
  // (for example, the test script failed to load) and `--wait-paused` would
  // wait forever. Some SDKs report a service URI even in that case, so the
  // check above is not enough on its own.
  final collected = await Future.any<Object?>([
    collection.then((_) => null),
    testProcessDone,
  ]);
  if (collected is int && collected != 0) {
    stderr.writeln(
      'Test process exited with code $collected before coverage collection '
      'finished.',
    );
    exit(collected);
  }
  await collection;
  return testProcessDone.timeout(
    const Duration(seconds: 3),
    onTimeout: () {
      for (final process in _allProcesses) {
        process.kill();
      }
      return testExitCode;
    },
  );
}

/// Writes intermediate `coverage.json` (`outJson`) for [_formatCoverage] to
/// format into `lcov.info` (`outLcov`), matching `collect_coverage.main` in
/// the VM flow.
Future<void> _writeWebCoverageJson(
  List<File> coverageFiles,
  Flags flags,
  String outJson,
) async {
  if (coverageFiles.isEmpty) {
    stderr.writeln(
      'warning: the test run produced no coverage files; the report will '
      'be empty.',
    );
    File(outJson).writeAsStringSync(
      jsonEncode({
        'type': 'CodeCoverage',
        'coverage': <Map<String, dynamic>>[],
      }),
    );
    return;
  }

  final scopes =
      (flags.scopeOutput.isEmpty
              ? getAllWorkspaceNames(flags.packageDir)
              : flags.scopeOutput)
          .toSet();

  final hitmap = await HitMap.parseFiles(
    coverageFiles,
    packagePath: flags.packageDir,
    scopeOutput: scopes,
  );

  if (hitmap.isEmpty) {
    stderr.writeln(
      'warning: no coverage data matched ${scopes.join(', ')}. '
      'Compiled web output may not map back to library sources.',
    );
  }

  final allCoverage = [
    for (final MapEntry(key: uriStr, value: map) in hitmap.entries)
      hitmapToJson(map, Uri.parse(uriStr)),
  ];

  final jsonOutput = jsonEncode({
    'type': 'CodeCoverage',
    'coverage': allCoverage,
  });
  File(outJson).writeAsStringSync(jsonOutput);
}

Future<int> _runWebTestsAndCollectCoverage(Flags flags, String outJson) async {
  await _validatePackageConfig(flags.packageDir);
  final tempDir = Directory.systemTemp.createTempSync('coverage_');
  try {
    final testArgs = [
      'test',
      '--coverage=${tempDir.path}',
      if (flags.branchCoverage) '--branch-coverage',
      '-p',
      flags.platform!,
      if (flags.testScript != 'test') flags.testScript,
      ...flags.rest,
    ];

    var exitCode = 0;
    try {
      await _dartRun(testArgs, workingDirectory: flags.packageDir);
    } on ProcessException catch (e) {
      exitCode = e.errorCode;
    }

    final coverageFiles = tempDir
        .listSync(recursive: true)
        .whereType<File>()
        .where((f) => f.path.endsWith('.json'))
        .toList();

    await _writeWebCoverageJson(coverageFiles, flags, outJson);
    return exitCode;
  } finally {
    try {
      tempDir.deleteSync(recursive: true);
    } catch (_) {}
  }
}

Future<void> _formatCoverage(Flags flags, String outJson, String outLcov) =>
    format_coverage.main([
      '--lcov',
      '--check-ignore',
      '--package=${flags.packageDir}',
      '-i',
      outJson,
      '-o',
      outLcov,
      if (flags.failUnder != null) '--fail-under=${flags.failUnder}',
    ]);

Future<void> main(List<String> arguments) async {
  final defaultOptions = CoverageOptionsProvider().coverageOptions;
  final flags = await parseArgs(arguments, defaultOptions);
  final outJson = path.join(flags.outDir, 'coverage.json');
  final outLcov = path.join(flags.outDir, 'lcov.info');

  if (!FileSystemEntity.isDirectorySync(flags.outDir)) {
    await Directory(flags.outDir).create(recursive: true);
  }

  _registerSignalHandlers();

  final isVm = flags.platform == null || flags.platform == 'vm';
  final exitCode = isVm
      ? await _runVmTestsAndCollectCoverage(flags, outJson)
      : await _runWebTestsAndCollectCoverage(flags, outJson);

  await _formatCoverage(flags, outJson, outLcov);
  exit(exitCode);
}