pkgs/coverage/test/test_with_coverage_test.dart

// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

@Tags(['integration'])
library;

import 'dart:convert';
import 'dart:io';

import 'package:path/path.dart' as p;
import 'package:test/test.dart';
import 'package:test_descriptor/test_descriptor.dart' as d;
import 'package:test_process/test_process.dart';

import 'test_util.dart';

// this package
final _pkgDir = p.absolute('');
final _testWithCoveragePath = p.join(_pkgDir, 'bin', 'test_with_coverage.dart');

// test package
final _testPkgDirPath = p.join(_pkgDir, 'test', 'test_with_coverage_package');

/// Override PUB_CACHE
///
/// Use a subdirectory different from `test/` just in case there is a problem
/// with the clean up. If other packages are present under the `test/`
/// subdirectory their tests may accidentally get run when running `dart test`
final _pubCachePathInTestPkgSubDir = p.join(_pkgDir, 'var', 'pub-cache');
final _env = {'PUB_CACHE': _pubCachePathInTestPkgSubDir};

const _testPackageName = 'coverage_integration_test_for_test_with_coverage';

Iterable<File> _dartFiles(String dir) =>
    Directory(p.join(_testPkgDirPath, dir)).listSync().whereType<File>();

String _fixTestFile(String content) =>
    content.replaceAll("import '../lib/", "import 'package:$_testPackageName/");

void main() {
  setUpAll(() async {
    for (var dir in const ['lib', 'test']) {
      await d.dir(dir, [
        for (var dartFile in _dartFiles(dir))
          d.file(
            p.basename(dartFile.path),
            _fixTestFile(dartFile.readAsStringSync()),
          ),
      ]).create();
    }

    var pubspecContent = File(
      p.join(_testPkgDirPath, 'pubspec.yaml'),
    ).readAsStringSync();

    expect(
      pubspecContent.replaceAll('\r\n', '\n'),
      contains(r'''
dependency_overrides:
  coverage:
    path: ../../
'''),
    );

    pubspecContent = pubspecContent.replaceFirst(
      'path: ../../',
      'path: $_pkgDir',
    );

    await d.file('pubspec.yaml', pubspecContent).create();

    final localPub = await _run(['pub', 'get']);
    await localPub.shouldExit(0);
  });

  tearDownAll(() async {
    if (Platform.isWindows) {
      final sandboxDir = Directory(d.sandbox);
      for (var i = 0; i < 5; i++) {
        await Future<void>.delayed(const Duration(milliseconds: 100));
        try {
          if (sandboxDir.existsSync()) {
            for (final entity in sandboxDir.listSync()) {
              entity.deleteSync(recursive: true);
            }
          }
          break;
        } catch (_) {}
      }
    }
  });

  test('dart run bin/test_with_coverage.dart -f', () async {
    final list = await _runTest(['run', _testWithCoveragePath, '-f']);

    final sources = list.sources();
    final functionHits = functionInfoFromSources(sources);

    expect(functionHits['package:$_testPackageName/validate_lib.dart'], {
      'product': 1,
      'sum': 1,
      'evaluateScore': 1,
    });
  });

  test('dart run bin/test_with_coverage.dart -f -- -N sum', () async {
    final list = await _runTest(
      ['run', _testWithCoveragePath, '-f'],
      extraArgs: ['--', '-N', 'sum'],
    );

    final sources = list.sources();
    final functionHits = functionInfoFromSources(sources);

    expect(functionHits['package:$_testPackageName/validate_lib.dart'], {
      'product': 0,
      'sum': 1,
      'evaluateScore': 0,
    }, reason: 'only `sum` tests should be run');
  });

  test('dart run coverage:test_with_coverage', () async {
    await _runTest(['run', 'coverage:test_with_coverage']);
  });

  test('dart pub global run coverage:test_with_coverage', () async {
    final globalPub = await _run([
      'pub',
      'global',
      'activate',
      '-s',
      'path',
      _pkgDir,
    ]);
    await globalPub.shouldExit(0);

    await _runTest(['pub', 'global', 'run', 'coverage:test_with_coverage']);
  });

  test('dart run bin/test_with_coverage.dart --fail-under succeeds'
      'when coverage meets threshold', () async {
    // This should pass as coverage=100% when all tests run.
    final process = await _run([
      'run',
      _testWithCoveragePath,
      '--fail-under=100',
      '--port',
      '0',
    ]);
    await process.shouldExit(0);
  });
  test('dart run bin/test_with_coverage.dart --fail-under fails'
      'when coverage is below threshold', () async {
    // This should throw an exit(1) as coverage =27.27% when only the `sum` test
    // is run i.e. out of 11 lines only 3 lines i.e. [5,7,8]will have hits>0.
    final process = await _run([
      'run',
      _testWithCoveragePath,
      '--fail-under=90',
      '--port',
      '0',
      '--',
      '-N',
      'sum',
    ]);
    await process.shouldExit(1);
  });
  test('dart run bin/test_with_coverage.dart -b --fail-under succeeds'
      'when coverage meets threshold', () async {
    // This should pass as total lines+branches covered=20 and total lines(11)+
    // branches covered(10)=21 => percentage_covered=95.23.
    final process = await _run([
      'run',
      _testWithCoveragePath,
      '-b',
      '--fail-under=90',
      '--port',
      '0',
    ]);
    await process.shouldExit(0);
  });
  test('dart run bin/test_with_coverage.dart -b --fail-under fails'
      'when coverage is below threshold', () async {
    final process = await _run([
      'run',
      _testWithCoveragePath,
      '-b',
      '--fail-under=99',
      '--port',
      '0',
    ]);
    await process.shouldExit(1);
  });

  test(
    'dart run bin/test_with_coverage.dart -p chrome',
    onPlatform: const {'windows': Skip('Chrome tests skipped on Windows')},
    timeout: const Timeout(Duration(minutes: 5)),
    () async {
      final list = await _runTest([
        'run',
        _testWithCoveragePath,
        '-p',
        'chrome',
      ]);
      final sources = list.sources();
      final lineHits = lineHitsFromSources(sources);

      final libHits = lineHits['package:$_testPackageName/validate_lib.dart'];
      expect(libHits, isNotNull);
      expect(libHits![6], greaterThanOrEqualTo(1));
      expect(libHits[14], greaterThanOrEqualTo(1));
      expect(libHits[22], greaterThanOrEqualTo(1));
    },
  );
}

Future<TestProcess> _run(List<String> args) => TestProcess.start(
  Platform.executable,
  args,
  workingDirectory: d.sandbox,
  environment: _env,
);

Future<List<Map<String, dynamic>>> _runTest(
  List<String> invokeArgs, {
  List<String>? extraArgs,
}) async {
  final process = await _run([...invokeArgs, '--port', '0', ...?extraArgs]);

  await process.shouldExit(0);

  await d.dir('coverage', [
    d.file('coverage.json', isNotEmpty),
    d.file('lcov.info', isNotEmpty),
  ]).validate();

  final coverageDataFile = File(p.join(d.sandbox, 'coverage', 'coverage.json'));

  final json = jsonDecode(coverageDataFile.readAsStringSync());

  return coverageDataFromJson(json as Map<String, dynamic>);
}