pkgs/test_reflective_loader/lib/test_reflective_loader.dart

// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:mirrors';

import 'package:test/test.dart' as test_package;

/// A marker annotation used to annotate test methods which are expected to fail
/// when asserts are enabled.
const Object assertFailingTest = _AssertFailingTest();

/// A marker annotation used to annotate test methods which are expected to
/// fail.
const Object failingTest = FailingTest();

/// A marker annotation used to instruct dart2js to keep reflection information
/// for the annotated classes.
const Object reflectiveTest = _ReflectiveTest();

/// A marker annotation used to annotate test methods that should be skipped.
const Object skippedTest = SkippedTest();

/// A marker annotation used to annotate "solo" groups and tests.
const Object soloTest = _SoloTest();

/// Whether the currently running reflective test is expected to fail.
bool get currentTestIsExpectedToFail =>
    Zone.current[_currentTestIsExpectedToFailKey] == true;

const _currentTestIsExpectedToFailKey = #currentTestIsExpectedToFail;

/// The current group stack of nested [defineReflectiveSuite] calls.
List<_Group> _currentGroupStack = [];

/// The root groups or tests created by [defineReflectiveSuite] or
/// [defineReflectiveTests] calls.
List<_GroupEntry> _rootGroupEntries = [];

/// Is `true` the application is running in the checked mode.
final bool _isCheckedMode = () {
  try {
    assert(false);
    return false;
  } catch (_) {
    return true;
  }
}();

/// Run the [define] function parameter that calls [defineReflectiveTests] to
/// add normal and "solo" tests, and also calls [defineReflectiveSuite] to
/// create embedded suites.  If the current suite is the top-level one, perform
/// check for "solo" groups and tests, and run all or only "solo" items.
void defineReflectiveSuite(void Function() define, {String? name}) {
  _addGroup(_Group(name), define);

  _addTestsIfTopLevelSuite();
}

/// Runs test methods existing in the given [type].
///
/// If there is a "solo" test method in the top-level suite, only "solo" methods
/// are run.
///
/// If there is a "solo" test type, only its test methods are run.
///
/// Otherwise all tests methods of all test types are run.
///
/// Each method is run with a new instance of [type].
/// So, [type] should have a default constructor.
///
/// If [type] declares method `setUp`, it methods will be invoked before any
/// test method invocation.
///
/// If [type] declares method `tearDown`, it will be invoked after any test
/// method invocation. If method returns [Future] to test some asynchronous
/// behavior, then `tearDown` will be invoked in `Future.complete`.
void defineReflectiveTests(Type type) {
  var classMirror = reflectClass(type);
  if (!classMirror.metadata.any(
    (InstanceMirror annotation) =>
        annotation.type.reflectedType == _ReflectiveTest,
  )) {
    var name = MirrorSystem.getName(classMirror.qualifiedName);
    throw Exception(
      'Class $name must have annotation "@reflectiveTest" '
      'in order to be run by runReflectiveTests.',
    );
  }

  var isSolo = _hasAnnotationInstance(classMirror, soloTest);
  var className = MirrorSystem.getName(classMirror.simpleName);

  _addGroup(
    _Group(
      className,
      solo: isSolo,
      location: classMirror.testLocation,
      classMirror: classMirror,
    ),
    () {
      classMirror.instanceMembers.forEach((
        Symbol symbol,
        MethodMirror memberMirror,
      ) {
        // we need only methods
        if (!memberMirror.isRegularMethod) {
          return;
        }
        // prepare information about the method
        var memberName = MirrorSystem.getName(symbol);
        var isTest = memberName.startsWith(RegExp('(solo_|fail_|skip_)*test_'));
        if (isTest) {
          var isSolo = memberName.startsWith('solo_') ||
              _hasAnnotationInstance(memberMirror, soloTest);
          var isSkipped = memberName.startsWith('skip_') ||
              _hasSkippedTestAnnotation(memberMirror);
          var expectFail = memberName.startsWith('fail_') ||
              memberName.startsWith('solo_fail_') ||
              _hasFailingTestAnnotation(memberMirror) ||
              _isCheckedMode && _hasAssertFailingTestAnnotation(memberMirror);
          var timeout =
              _getAnnotationInstance(memberMirror, TestTimeout) as TestTimeout?;

          _addTest(
            _Test(
              memberName,
              timeout: timeout?._timeout,
              location: memberMirror.testLocation,
              solo: isSolo,
              skip: isSkipped,
              () => runZoned(
                () => expectFail
                    ? _runFailingTest(classMirror, symbol)
                    : _runTest(classMirror, symbol),
                zoneValues: {_currentTestIsExpectedToFailKey: expectFail},
              ),
            ),
          );
        }
      });
    },
  );

  _addTestsIfTopLevelSuite();
}

/// If we're back at the top level ([_currentGroupStack] is empty), registers
/// all known groups and tests by calling [test_package.group] and
/// [test_package.test] appropriately.
void _addTestsIfTopLevelSuite() {
  if (_currentGroupStack.isNotEmpty) return;

  void addGroupsAndTests(List<_GroupEntry> entries) {
    for (var entry in entries) {
      switch (entry) {
        case _Group group:
          // Only add groups if they have names, otherwise just add their
          // children directly.
          if (group.name != null) {
            test_package.group(
              group.name,
              location: group.location,
              // ignore: deprecated_member_use, invalid_use_of_do_not_submit_member
              solo: group.solo,
              () {
                // If this group is a class, it may have class-wide
                // setUp/tearDown. Only add the methods if they exist so that
                // they don't show in the results/IDE if not.
                if (group.classMirror case var classMirror?) {
                  var setUp = _getClosureMember(classMirror, #setUpClass);
                  if (setUp != null) {
                    test_package.setUpAll(group.ensureSetUpClass,
                        location: setUp.function.testLocation);
                  }

                  var tearDown = _getClosureMember(classMirror, #tearDownClass);
                  if (tearDown != null) {
                    test_package.tearDownAll(group.tearDownClass,
                        location: tearDown.function.testLocation);
                  }
                }
                addGroupsAndTests(group.children);
              },
            );
          } else {
            addGroupsAndTests(group.children);
          }
          break;
        case _Test test:
          test_package.test(
            test.name,
            timeout: test.timeout,
            location: test.location,
            // ignore: deprecated_member_use, invalid_use_of_do_not_submit_member
            solo: test.solo,
            skip: test.skip,
            test.function,
          );
          break;
      }
    }
  }

  addGroupsAndTests(_rootGroupEntries);
  _rootGroupEntries.clear();
}

Object? _getAnnotationInstance(DeclarationMirror declaration, Type type) {
  for (var annotation in declaration.metadata) {
    if ((annotation.reflectee as Object).runtimeType == type) {
      return annotation.reflectee;
    }
  }
  return null;
}

bool _hasAnnotationInstance(DeclarationMirror declaration, Object instance) =>
    declaration.metadata.any(
      (InstanceMirror annotation) => identical(annotation.reflectee, instance),
    );

bool _hasAssertFailingTestAnnotation(MethodMirror method) =>
    _hasAnnotationInstance(method, assertFailingTest);

bool _hasFailingTestAnnotation(MethodMirror method) =>
    _hasAnnotationInstance(method, failingTest);

bool _hasSkippedTestAnnotation(MethodMirror method) =>
    _hasAnnotationInstance(method, skippedTest);

/// Invokes [symbol] on [objectMirror] if it is a closure and returns the
/// result.
Future<Object?> _invokeSymbolIfExists(
  ObjectMirror objectMirror,
  Symbol symbol,
) {
  Object? invocationResult;
  var closure = _getClosureMember(objectMirror, symbol);
  invocationResult = closure?.apply([]).reflectee;
  return Future.value(invocationResult);
}

/// Gets [symbol] from [objectMirror] if it is a closure.
ClosureMirror? _getClosureMember(ObjectMirror objectMirror, Symbol symbol) {
  try {
    var member = objectMirror.getField(symbol);
    return member is ClosureMirror ? member : null;
    // ignore: avoid_catching_errors
  } on NoSuchMethodError {
    // ignore: empty_catches
  }

  return null;
}

/// Adds a group to the current stack and executes [define] for child group
/// or tests definitions.
void _addGroup(_Group group, void Function() define) {
  var parentCollection =
      _currentGroupStack.lastOrNull?.children ?? _rootGroupEntries;
  parentCollection.add(group);
  _currentGroupStack.add(group);
  try {
    define();
  } finally {
    _currentGroupStack.removeLast();
  }
}

/// Adds a test to the current group (or as a root test if there is no current
/// group).
void _addTest(_Test test) {
  var parentCollection =
      _currentGroupStack.lastOrNull?.children ?? _rootGroupEntries;
  parentCollection.add(test);
}

/// Run a test that is expected to fail, and confirm that it fails.
///
/// This properly handles the following cases:
/// - The test fails by throwing an exception
/// - The test returns a future which completes with an error.
/// - An exception is thrown to the zone handler from a timer task.
Future<void> _runFailingTest(ClassMirror classMirror, Symbol symbol) async {
  _FailedTestResult? result;

  await runZonedGuarded(
    () {
      // ignore: void_checks
      return Future.sync(() => _runTest(classMirror, symbol)).then<void>((_) {
        // We can't throw async exceptions inside here because `runZoneGuarded`
        // will never complete (see docs on `runZonedGuarded`), so we need to
        // capture this state and throw later if there wasn't otherwise an
        // exception.

        // If we didn't already have a failure (eg. an unawaited exception) then
        // this successful completion is an unexpected pass state.
        result ??= _FailedTestResult.pass;
      }).catchError((Object e) {
        // an awaited exception is always expected failure.
        result = _FailedTestResult.expectedFail;
      });
    },
    (e, st) {
      result = _FailedTestResult.expectedFail;
    },
  );

  // We can safely throw exceptions back outside of the error zone.
  if (result == _FailedTestResult.pass) {
    throw test_package.TestFailure('Test passed - expected to fail.');
  }
}

Future<void> _runTest(ClassMirror classMirror, Symbol symbol) async {
  var instanceMirror = classMirror.newInstance(const Symbol(''), []);
  try {
    await _invokeSymbolIfExists(instanceMirror, #setUp);
    await instanceMirror.invoke(symbol, []).reflectee;
  } finally {
    await _invokeSymbolIfExists(instanceMirror, #tearDown);
  }
}

/// The result of a test that was expected to fail.
enum _FailedTestResult {
  /// The test (unexpectedly) passed.
  pass,

  /// The test failed as expected.
  expectedFail,
}

/// A marker annotation used to annotate test methods which are expected to
/// fail.
class FailingTest {
  /// Initialize this annotation with the given arguments.
  ///
  /// [issue] is a full URI describing the failure and used for tracking.
  /// [reason] is a free form textual description.
  const FailingTest({String? issue, String? reason});
}

/// A marker annotation used to annotate test methods which are skipped.
class SkippedTest {
  /// Initialize this annotation with the given arguments.
  ///
  /// [issue] is a full URI describing the failure and used for tracking.
  /// [reason] is a free form textual description.
  const SkippedTest({String? issue, String? reason});
}

/// A marker annotation used to annotate test methods with additional timeout
/// information.
class TestTimeout {
  final test_package.Timeout _timeout;

  /// Initialize this annotation with the given timeout.
  const TestTimeout(test_package.Timeout timeout) : _timeout = timeout;
}

/// A marker annotation used to annotate test methods which are expected to fail
/// when asserts are enabled.
class _AssertFailingTest {
  const _AssertFailingTest();
}

/// A marker annotation used to instruct dart2js to keep reflection information
/// for the annotated classes.
class _ReflectiveTest {
  const _ReflectiveTest();
}

/// A marker annotation used to annotate "solo" groups and tests.
class _SoloTest {
  const _SoloTest();
}

abstract class _GroupEntry {
  final String? name;
  final test_package.TestLocation? location;
  final bool solo;

  _GroupEntry(this.name, {this.location, this.solo = false});
}

/// Information about a test group which could be from a call to
/// [defineReflectiveSuite] with a `name`, or a test class itself.
class _Group extends _GroupEntry {
  final List<_GroupEntry> children = [];
  final ClassMirror? classMirror;
  Future<Object?>? _setUpCompletion;

  _Group(super.name, {super.location, super.solo, this.classMirror});

  /// Runs group-wide setup if it has not been started yet,
  /// ensuring it only runs once for a group. Set up runs and
  /// completes before any test of the group runs
  Future<Object?> ensureSetUpClass() {
    if (classMirror == null) return Future.value();
    return _setUpCompletion ??= _invokeSymbolIfExists(
      classMirror!,
      #setUpClass,
    );
  }

  /// Runs group-wide tear down if [ensureSetUpClass] was called at least once.
  /// Must be called once and only called after all tests of the group have
  /// completed
  Future<Object?>? tearDownClass() {
    if (classMirror == null) return null;
    return _setUpCompletion != null
        ? _invokeSymbolIfExists(classMirror!, #tearDownClass)
        : null;
  }
}

/// Information about a test created for a method of a class with
/// [defineReflectiveTests].
class _Test extends _GroupEntry {
  final FutureOr<Object?>? Function() function;
  final bool skip;
  final test_package.Timeout? timeout;

  _Test(
    super.name,
    this.function, {
    required super.location,
    required super.solo,
    required this.skip,
    required this.timeout,
  });
}

extension on DeclarationMirror {
  test_package.TestLocation? get testLocation {
    if (location case var location?) {
      return test_package.TestLocation(
        location.sourceUri,
        location.line,
        location.column,
      );
    } else {
      return null;
    }
  }
}