pkgs/test/test/utils.dart

// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:collection';

import 'package:boolean_selector/boolean_selector.dart';
import 'package:glob/glob.dart';
import 'package:test_api/src/backend/declarer.dart';
import 'package:test_api/src/backend/group.dart';
import 'package:test_api/src/backend/group_entry.dart';
import 'package:test_api/src/backend/invoker.dart';
import 'package:test_api/src/backend/live_test.dart';
import 'package:test_api/src/backend/metadata.dart';
import 'package:test_api/src/backend/platform_selector.dart';
import 'package:test_api/src/backend/runtime.dart';
import 'package:test_api/src/backend/state.dart';
import 'package:test_api/src/backend/suite.dart';
import 'package:test_api/src/backend/suite_platform.dart';
import 'package:test_core/src/runner/application_exception.dart';
import 'package:test_core/src/runner/configuration.dart';
import 'package:test_core/src/runner/configuration/custom_runtime.dart';
import 'package:test_core/src/runner/configuration/runtime_settings.dart';
import 'package:test_core/src/runner/engine.dart';
import 'package:test_core/src/runner/load_suite.dart';
import 'package:test_core/src/runner/plugin/environment.dart';
import 'package:test_core/src/runner/runner_suite.dart';
import 'package:test_core/src/runner/runtime_selection.dart';
import 'package:test_core/src/runner/suite.dart';
import 'package:test/test.dart';

/// A dummy suite platform to use for testing suites.
final suitePlatform = SuitePlatform(Runtime.vm);

// The last state change detected via [expectStates].
State? lastState;

/// Asserts that exactly [states] will be emitted via [liveTest.onStateChange].
///
/// The most recent emitted state is stored in [_lastState].
void expectStates(LiveTest liveTest, Iterable<State> statesIter) {
  var states = Queue.from(statesIter);
  liveTest.onStateChange.listen(expectAsync1((state) {
    lastState = state;
    expect(state, equals(states.removeFirst()));
  }, count: states.length, max: states.length));
}

/// Asserts that errors will be emitted via [liveTest.onError] that match
/// [validators], in order.
void expectErrors(LiveTest liveTest, Iterable<Function> validatorsIter) {
  var validators = Queue.from(validatorsIter);
  liveTest.onError.listen(expectAsync1((error) {
    validators.removeFirst()(error.error);
  }, count: validators.length, max: validators.length));
}

/// Asserts that [liveTest] will have a single failure with message `"oh no"`.
void expectSingleFailure(LiveTest liveTest) {
  expectStates(liveTest, [
    const State(Status.running, Result.success),
    const State(Status.complete, Result.failure)
  ]);

  expectErrors(liveTest, [
    (error) {
      expect(lastState!.status, equals(Status.complete));
      expect(error, isTestFailure('oh no'));
    }
  ]);
}

/// Asserts that [liveTest] will have a single error, the string `"oh no"`.
void expectSingleError(LiveTest liveTest) {
  expectStates(liveTest, [
    const State(Status.running, Result.success),
    const State(Status.complete, Result.error)
  ]);

  expectErrors(liveTest, [
    (error) {
      expect(lastState!.status, equals(Status.complete));
      expect(error, equals('oh no'));
    }
  ]);
}

/// Returns a matcher that matches a callback or Future that throws a
/// [TestFailure] with the given [message].
///
/// [message] can be a string or a [Matcher].
Matcher throwsTestFailure(message) => throwsA(isTestFailure(message));

/// Returns a matcher that matches a [TestFailure] with the given [message].
///
/// [message] can be a string or a [Matcher].
Matcher isTestFailure(message) => const TypeMatcher<TestFailure>()
    .having((e) => e.message, 'message', message);

/// Returns a matcher that matches a [ApplicationException] with the given
/// [message].
///
/// [message] can be a string or a [Matcher].
Matcher isApplicationException(message) =>
    const TypeMatcher<ApplicationException>()
        .having((e) => e.message, 'message', message);

/// Returns a local [LiveTest] that runs [body].
LiveTest createTest(dynamic Function() body) {
  var test = LocalTest('test', Metadata(chainStackTraces: true), body);
  var suite = Suite(Group.root([test]), suitePlatform);
  return test.load(suite);
}

/// Runs [body] as a test.
///
/// Once it completes, returns the [LiveTest] used to run it.
Future<LiveTest> runTestBody(dynamic Function() body) async {
  var liveTest = createTest(body);
  await liveTest.run();
  return liveTest;
}

/// Asserts that [liveTest] has completed and passed.
///
/// If the test had any errors, they're surfaced nicely into the outer test.
void expectTestPassed(LiveTest liveTest) {
  // Since the test is expected to pass, we forward any current or future errors
  // to the outer test, because they're definitely unexpected.
  for (var error in liveTest.errors) {
    registerException(error.error, error.stackTrace);
  }
  liveTest.onError.listen((error) {
    registerException(error.error, error.stackTrace);
  });

  expect(liveTest.state.status, equals(Status.complete));
  expect(liveTest.state.result, equals(Result.success));
}

/// Asserts that [liveTest] failed with a single [TestFailure] whose message
/// matches [message].
void expectTestFailed(LiveTest liveTest, message) {
  expect(liveTest.state.status, equals(Status.complete));
  expect(liveTest.state.result, equals(Result.failure));
  expect(liveTest.errors, hasLength(1));
  expect(liveTest.errors.first.error, isTestFailure(message));
}

/// Assert that the [test] callback causes a test to block until [stopBlocking]
/// is called at some later time.
///
/// [stopBlocking] is passed the return value of [test].
Future<void> expectTestBlocks(
    dynamic Function() test, dynamic Function(dynamic) stopBlocking) async {
  late LiveTest liveTest;
  late Future<void> future;
  liveTest = createTest(() {
    var value = test();
    future = pumpEventQueue().then((_) {
      expect(liveTest.state.status, equals(Status.running));
      stopBlocking(value);
    });
  });

  await liveTest.run();
  expectTestPassed(liveTest);
  // Ensure that the outer test doesn't complete until the inner future
  // completes.
  return future;
}

/// Runs [body] with a declarer, runs all the declared tests, and asserts that
/// they pass.
///
/// This is typically used to run multiple tests where later tests make
/// assertions about the results of previous ones.
Future<void> expectTestsPass(void Function() body) async {
  var engine = declareEngine(body);
  var success = await engine.run();

  for (var test in engine.liveTests) {
    expectTestPassed(test);
  }

  expect(success, isTrue);
}

/// Runs [body] with a declarer and returns the declared entries.
List<GroupEntry> declare(void Function() body) {
  var declarer = Declarer()..declare(body);
  return declarer.build().entries;
}

/// Runs [body] with a declarer and returns an engine that runs those tests.
Engine declareEngine(void Function() body,
    {bool runSkipped = false, String? coverage}) {
  var declarer = Declarer()..declare(body);
  return Engine.withSuites([
    RunnerSuite(
        const PluginEnvironment(),
        SuiteConfiguration.runSkipped(runSkipped),
        declarer.build(),
        suitePlatform)
  ], coverage: coverage);
}

/// Returns a [RunnerSuite] with a default environment and configuration.
RunnerSuite runnerSuite(Group root) => RunnerSuite(
    const PluginEnvironment(), SuiteConfiguration.empty, root, suitePlatform);

/// Returns a [LoadSuite] with a default configuration.
LoadSuite loadSuite(String name, FutureOr<RunnerSuite> Function() body) =>
    LoadSuite(name, SuiteConfiguration.empty, suitePlatform, body);

SuiteConfiguration suiteConfiguration(
        {bool? allowTestRandomization,
        bool? jsTrace,
        bool? runSkipped,
        Iterable<String>? dart2jsArgs,
        String? precompiledPath,
        Iterable<Pattern>? patterns,
        Iterable<RuntimeSelection>? runtimes,
        BooleanSelector? includeTags,
        BooleanSelector? excludeTags,
        Map<BooleanSelector, SuiteConfiguration>? tags,
        Map<PlatformSelector, SuiteConfiguration>? onPlatform,

        // Test-level configuration
        Timeout? timeout,
        bool? verboseTrace,
        bool? chainStackTraces,
        bool? skip,
        int? retry,
        String? skipReason,
        PlatformSelector? testOn,
        Iterable<String>? addTags}) =>
    SuiteConfiguration(
        allowTestRandomization: allowTestRandomization,
        jsTrace: jsTrace,
        runSkipped: runSkipped,
        dart2jsArgs: dart2jsArgs,
        precompiledPath: precompiledPath,
        patterns: patterns,
        runtimes: runtimes,
        includeTags: includeTags,
        excludeTags: excludeTags,
        tags: tags,
        onPlatform: onPlatform,
        timeout: timeout,
        verboseTrace: verboseTrace,
        chainStackTraces: chainStackTraces,
        skip: skip,
        retry: retry,
        skipReason: skipReason,
        testOn: testOn,
        addTags: addTags);

Configuration configuration(
        {bool? help,
        String? customHtmlTemplatePath,
        bool? version,
        bool? pauseAfterLoad,
        bool? debug,
        bool? color,
        String? configurationPath,
        String? dart2jsPath,
        String? reporter,
        Map<String, String>? fileReporters,
        String? coverage,
        int? pubServePort,
        int? concurrency,
        int? shardIndex,
        int? totalShards,
        Iterable<String>? paths,
        Iterable<String>? foldTraceExcept,
        Iterable<String>? foldTraceOnly,
        Glob? filename,
        Iterable<String>? chosenPresets,
        Map<String, Configuration>? presets,
        Map<String, RuntimeSettings>? overrideRuntimes,
        Map<String, CustomRuntime>? defineRuntimes,
        bool? noRetry,
        bool? useDataIsolateStrategy,

        // Suite-level configuration
        bool? allowTestRandomization,
        bool? jsTrace,
        bool? runSkipped,
        Iterable<String>? dart2jsArgs,
        String? precompiledPath,
        Iterable<Pattern>? patterns,
        Iterable<RuntimeSelection>? runtimes,
        BooleanSelector? includeTags,
        BooleanSelector? excludeTags,
        Map<BooleanSelector, SuiteConfiguration>? tags,
        Map<PlatformSelector, SuiteConfiguration>? onPlatform,
        int? testRandomizeOrderingSeed,

        // Test-level configuration
        Timeout? timeout,
        bool? verboseTrace,
        bool? chainStackTraces,
        bool? skip,
        int? retry,
        String? skipReason,
        PlatformSelector? testOn,
        Iterable<String>? addTags}) =>
    Configuration(
        help: help,
        customHtmlTemplatePath: customHtmlTemplatePath,
        version: version,
        pauseAfterLoad: pauseAfterLoad,
        debug: debug,
        color: color,
        configurationPath: configurationPath,
        dart2jsPath: dart2jsPath,
        reporter: reporter,
        fileReporters: fileReporters,
        coverage: coverage,
        pubServePort: pubServePort,
        concurrency: concurrency,
        shardIndex: shardIndex,
        totalShards: totalShards,
        paths: paths,
        foldTraceExcept: foldTraceExcept,
        foldTraceOnly: foldTraceOnly,
        filename: filename,
        chosenPresets: chosenPresets,
        presets: presets,
        overrideRuntimes: overrideRuntimes,
        defineRuntimes: defineRuntimes,
        noRetry: noRetry,
        useDataIsolateStrategy: useDataIsolateStrategy,
        allowTestRandomization: allowTestRandomization,
        jsTrace: jsTrace,
        runSkipped: runSkipped,
        dart2jsArgs: dart2jsArgs,
        precompiledPath: precompiledPath,
        patterns: patterns,
        runtimes: runtimes,
        includeTags: includeTags,
        excludeTags: excludeTags,
        tags: tags,
        onPlatform: onPlatform,
        testRandomizeOrderingSeed: testRandomizeOrderingSeed,
        timeout: timeout,
        verboseTrace: verboseTrace,
        chainStackTraces: chainStackTraces,
        skip: skip,
        retry: retry,
        skipReason: skipReason,
        testOn: testOn,
        addTags: addTags);