Merge package:test_reflective_loader into the tools monorepo
diff --git a/pkgs/test_reflective_loader/.github/dependabot.yml b/pkgs/test_reflective_loader/.github/dependabot.yml new file mode 100644 index 0000000..cde02ad --- /dev/null +++ b/pkgs/test_reflective_loader/.github/dependabot.yml
@@ -0,0 +1,15 @@ +# Dependabot configuration file. +# See https://docs.github.com/en/code-security/dependabot/dependabot-version-updates +version: 2 + +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + labels: + - autosubmit + groups: + github-actions: + patterns: + - "*"
diff --git a/pkgs/test_reflective_loader/.github/workflows/build.yaml b/pkgs/test_reflective_loader/.github/workflows/build.yaml new file mode 100644 index 0000000..2c15d95 --- /dev/null +++ b/pkgs/test_reflective_loader/.github/workflows/build.yaml
@@ -0,0 +1,33 @@ +name: Dart + +on: + pull_request: + push: + branches: + - master + schedule: + # “At 00:00 (UTC) on Sunday.” + - cron: '0 0 * * 0' + +env: + PUB_ENVIRONMENT: bot.github + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sdk: [dev, 3.1] + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94 + with: + sdk: ${{ matrix.sdk }} + + - run: dart pub get + - name: dart format + run: dart format --output=none --set-exit-if-changed . + - run: dart analyze --fatal-infos + - run: dart test
diff --git a/pkgs/test_reflective_loader/.gitignore b/pkgs/test_reflective_loader/.gitignore new file mode 100644 index 0000000..2a2c261 --- /dev/null +++ b/pkgs/test_reflective_loader/.gitignore
@@ -0,0 +1,11 @@ +.buildlog +.DS_Store +.idea +.dart_tool/ +.pub/ +.project +.settings/ +build/ +packages +.packages +pubspec.lock
diff --git a/pkgs/test_reflective_loader/AUTHORS b/pkgs/test_reflective_loader/AUTHORS new file mode 100644 index 0000000..e8063a8 --- /dev/null +++ b/pkgs/test_reflective_loader/AUTHORS
@@ -0,0 +1,6 @@ +# Below is a list of people and organizations that have contributed +# to the project. Names should be added to the list like so: +# +# Name/Organization <email address> + +Google Inc.
diff --git a/pkgs/test_reflective_loader/CHANGELOG.md b/pkgs/test_reflective_loader/CHANGELOG.md new file mode 100644 index 0000000..5fd4c55 --- /dev/null +++ b/pkgs/test_reflective_loader/CHANGELOG.md
@@ -0,0 +1,71 @@ +## 0.2.3-wip + +- Require Dart `^3.1.0`. + +## 0.2.2 + +- Update to package:lints 2.0.0 and move it to a dev dependency. + +## 0.2.1 + +- Use package:lints for analysis. +- Populate the pubspec `repository` field. + +## 0.2.0 + +- Stable null safety release. + +## 0.2.0-nullsafety.0 + +- Migrate to the null safety language feature. + +## 0.1.9 + +- Add `@SkippedTest` annotation and `skip_test` prefix. + +## 0.1.8 + +- Update `FailingTest` to add named parameters `issue` and `reason`. + +## 0.1.7 + +- Update documentation comments. +- Remove `@MirrorsUsed` annotation on `dart:mirrors`. + +## 0.1.6 + +- Make `FailingTest` public, with the URI of the issue that causes + the test to break. + +## 0.1.5 + +- Set max SDK version to `<3.0.0`, and adjust other dependencies. + +## 0.1.3 + +- Fix `@failingTest` to fail when the test passes. + +## 0.1.2 + +- Update the pubspec `dependencies` section to include `package:test` + +## 0.1.1 + +- For `@failingTest` tests, properly handle when the test fails by throwing an + exception in a timer task +- Analyze this package in strong mode + +## 0.1.0 + +- Switched from 'package:unittest' to 'package:test'. +- Since 'package:test' does not define 'solo_test', in order to keep this + functionality, `defineReflectiveSuite` must be used to wrap all + `defineReflectiveTests` invocations. + +## 0.0.4 + +- Added @failingTest, @assertFailingTest and @soloTest annotations. + +## 0.0.1 + +- Initial version
diff --git a/pkgs/test_reflective_loader/CONTRIBUTING.md b/pkgs/test_reflective_loader/CONTRIBUTING.md new file mode 100644 index 0000000..6f5e0ea --- /dev/null +++ b/pkgs/test_reflective_loader/CONTRIBUTING.md
@@ -0,0 +1,33 @@ +Want to contribute? Great! First, read this page (including the small print at +the end). + +### Before you contribute +Before we can use your code, you must sign the +[Google Individual Contributor License Agreement](https://cla.developers.google.com/about/google-individual) +(CLA), which you can do online. The CLA is necessary mainly because you own the +copyright to your changes, even after your contribution becomes part of our +codebase, so we need your permission to use and distribute your code. We also +need to be sure of various other things—for instance that you'll tell us if you +know that your code infringes on other people's patents. You don't have to sign +the CLA until after you've submitted your code for review and a member has +approved it, but you must do it before we can put your code into our codebase. + +Before you start working on a larger contribution, you should get in touch with +us first through the issue tracker with your idea so that we can help out and +possibly guide you. Coordinating up front makes it much easier to avoid +frustration later on. + +### Code reviews +All submissions, including submissions by project members, require review. + +### File headers +All files in the project must start with the following header. + + // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file + // for details. All rights reserved. Use of this source code is governed by a + // BSD-style license that can be found in the LICENSE file. + +### The small print +Contributions made by corporations are covered by a different agreement than the +one above, the +[Software Grant and Corporate Contributor License Agreement](https://developers.google.com/open-source/cla/corporate).
diff --git a/pkgs/test_reflective_loader/LICENSE b/pkgs/test_reflective_loader/LICENSE new file mode 100644 index 0000000..633672a --- /dev/null +++ b/pkgs/test_reflective_loader/LICENSE
@@ -0,0 +1,27 @@ +Copyright 2015, the Dart project authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/pkgs/test_reflective_loader/README.md b/pkgs/test_reflective_loader/README.md new file mode 100644 index 0000000..f7d07db --- /dev/null +++ b/pkgs/test_reflective_loader/README.md
@@ -0,0 +1,27 @@ +[](https://github.com/dart-lang/test_reflective_loader/actions) +[](https://pub.dev/packages/test_reflective_loader) + +Support for discovering tests and test suites using reflection. + +This package follows the xUnit style where each class is a test suite, and each +method with the name prefix `test_` is a single test. + +Methods with names starting with `test_` are run using the `test()` function with +the corresponding name. If the class defines methods `setUp()` or `tearDown()`, +they are executed before / after each test correspondingly, even if the test fails. + +Methods with names starting with `solo_test_` are run using the `solo_test()` function. + +Methods with names starting with `fail_` are expected to fail. + +Methods with names starting with `solo_fail_` are run using the `solo_test()` function +and expected to fail. + +Method returning `Future` class instances are asynchronous, so `tearDown()` is +executed after the returned `Future` completes. + +## Features and bugs + +Please file feature requests and bugs at the [issue tracker][tracker]. + +[tracker]: https://github.com/dart-lang/test_reflective_loader/issues
diff --git a/pkgs/test_reflective_loader/analysis_options.yaml b/pkgs/test_reflective_loader/analysis_options.yaml new file mode 100644 index 0000000..ea61158 --- /dev/null +++ b/pkgs/test_reflective_loader/analysis_options.yaml
@@ -0,0 +1,5 @@ +include: package:dart_flutter_team_lints/analysis_options.yaml + +linter: + rules: + - public_member_api_docs
diff --git a/pkgs/test_reflective_loader/lib/test_reflective_loader.dart b/pkgs/test_reflective_loader/lib/test_reflective_loader.dart new file mode 100644 index 0000000..cb69bf3 --- /dev/null +++ b/pkgs/test_reflective_loader/lib/test_reflective_loader.dart
@@ -0,0 +1,354 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:mirrors'; + +import 'package:test/test.dart' as test_package; + +/// A marker annotation used to annotate test methods which are expected to fail +/// when asserts are enabled. +const Object assertFailingTest = _AssertFailingTest(); + +/// A marker annotation used to annotate test methods which are expected to +/// fail. +const Object failingTest = FailingTest(); + +/// A marker annotation used to instruct dart2js to keep reflection information +/// for the annotated classes. +const Object reflectiveTest = _ReflectiveTest(); + +/// A marker annotation used to annotate test methods that should be skipped. +const Object skippedTest = SkippedTest(); + +/// A marker annotation used to annotate "solo" groups and tests. +const Object soloTest = _SoloTest(); + +final List<_Group> _currentGroups = <_Group>[]; +int _currentSuiteLevel = 0; +String _currentSuiteName = ''; + +/// Is `true` the application is running in the checked mode. +final bool _isCheckedMode = () { + try { + assert(false); + return false; + } catch (_) { + return true; + } +}(); + +/// Run the [define] function parameter that calls [defineReflectiveTests] to +/// add normal and "solo" tests, and also calls [defineReflectiveSuite] to +/// create embedded suites. If the current suite is the top-level one, perform +/// check for "solo" groups and tests, and run all or only "solo" items. +void defineReflectiveSuite(void Function() define, {String name = ''}) { + var groupName = _currentSuiteName; + _currentSuiteLevel++; + try { + _currentSuiteName = _combineNames(_currentSuiteName, name); + define(); + } finally { + _currentSuiteName = groupName; + _currentSuiteLevel--; + } + _addTestsIfTopLevelSuite(); +} + +/// Runs test methods existing in the given [type]. +/// +/// If there is a "solo" test method in the top-level suite, only "solo" methods +/// are run. +/// +/// If there is a "solo" test type, only its test methods are run. +/// +/// Otherwise all tests methods of all test types are run. +/// +/// Each method is run with a new instance of [type]. +/// So, [type] should have a default constructor. +/// +/// If [type] declares method `setUp`, it methods will be invoked before any +/// test method invocation. +/// +/// If [type] declares method `tearDown`, it will be invoked after any test +/// method invocation. If method returns [Future] to test some asynchronous +/// behavior, then `tearDown` will be invoked in `Future.complete`. +void defineReflectiveTests(Type type) { + var classMirror = reflectClass(type); + if (!classMirror.metadata.any((InstanceMirror annotation) => + annotation.type.reflectedType == _ReflectiveTest)) { + var name = MirrorSystem.getName(classMirror.qualifiedName); + throw Exception('Class $name must have annotation "@reflectiveTest" ' + 'in order to be run by runReflectiveTests.'); + } + + _Group group; + { + var isSolo = _hasAnnotationInstance(classMirror, soloTest); + var className = MirrorSystem.getName(classMirror.simpleName); + group = _Group(isSolo, _combineNames(_currentSuiteName, className)); + _currentGroups.add(group); + } + + classMirror.instanceMembers + .forEach((Symbol symbol, MethodMirror memberMirror) { + // we need only methods + if (!memberMirror.isRegularMethod) { + return; + } + // prepare information about the method + var memberName = MirrorSystem.getName(symbol); + var isSolo = memberName.startsWith('solo_') || + _hasAnnotationInstance(memberMirror, soloTest); + // test_ + if (memberName.startsWith('test_')) { + if (_hasSkippedTestAnnotation(memberMirror)) { + group.addSkippedTest(memberName); + } else { + group.addTest(isSolo, memberName, memberMirror, () { + if (_hasFailingTestAnnotation(memberMirror) || + _isCheckedMode && _hasAssertFailingTestAnnotation(memberMirror)) { + return _runFailingTest(classMirror, symbol); + } else { + return _runTest(classMirror, symbol); + } + }); + } + return; + } + // solo_test_ + if (memberName.startsWith('solo_test_')) { + group.addTest(true, memberName, memberMirror, () { + return _runTest(classMirror, symbol); + }); + } + // fail_test_ + if (memberName.startsWith('fail_')) { + group.addTest(isSolo, memberName, memberMirror, () { + return _runFailingTest(classMirror, symbol); + }); + } + // solo_fail_test_ + if (memberName.startsWith('solo_fail_')) { + group.addTest(true, memberName, memberMirror, () { + return _runFailingTest(classMirror, symbol); + }); + } + // skip_test_ + if (memberName.startsWith('skip_test_')) { + group.addSkippedTest(memberName); + } + }); + + // Support for the case of missing enclosing [defineReflectiveSuite]. + _addTestsIfTopLevelSuite(); +} + +/// If the current suite is the top-level one, add tests to the `test` package. +void _addTestsIfTopLevelSuite() { + if (_currentSuiteLevel == 0) { + void runTests({required bool allGroups, required bool allTests}) { + for (var group in _currentGroups) { + if (allGroups || group.isSolo) { + for (var test in group.tests) { + if (allTests || test.isSolo) { + test_package.test(test.name, test.function, + timeout: test.timeout, skip: test.isSkipped); + } + } + } + } + } + + if (_currentGroups.any((g) => g.hasSoloTest)) { + runTests(allGroups: true, allTests: false); + } else if (_currentGroups.any((g) => g.isSolo)) { + runTests(allGroups: false, allTests: true); + } else { + runTests(allGroups: true, allTests: true); + } + _currentGroups.clear(); + } +} + +/// Return the combination of the [base] and [addition] names. +/// If any other two is `null`, then the other one is returned. +String _combineNames(String base, String addition) { + if (base.isEmpty) { + return addition; + } else if (addition.isEmpty) { + return base; + } else { + return '$base | $addition'; + } +} + +Object? _getAnnotationInstance(DeclarationMirror declaration, Type type) { + for (var annotation in declaration.metadata) { + if ((annotation.reflectee as Object).runtimeType == type) { + return annotation.reflectee; + } + } + return null; +} + +bool _hasAnnotationInstance(DeclarationMirror declaration, Object instance) => + declaration.metadata.any((InstanceMirror annotation) => + identical(annotation.reflectee, instance)); + +bool _hasAssertFailingTestAnnotation(MethodMirror method) => + _hasAnnotationInstance(method, assertFailingTest); + +bool _hasFailingTestAnnotation(MethodMirror method) => + _hasAnnotationInstance(method, failingTest); + +bool _hasSkippedTestAnnotation(MethodMirror method) => + _hasAnnotationInstance(method, skippedTest); + +Future<Object?> _invokeSymbolIfExists( + InstanceMirror instanceMirror, Symbol symbol) { + Object? invocationResult; + InstanceMirror? closure; + try { + closure = instanceMirror.getField(symbol); + // ignore: avoid_catching_errors + } on NoSuchMethodError { + // ignore + } + + if (closure is ClosureMirror) { + invocationResult = closure.apply([]).reflectee; + } + return Future.value(invocationResult); +} + +/// Run a test that is expected to fail, and confirm that it fails. +/// +/// This properly handles the following cases: +/// - The test fails by throwing an exception +/// - The test returns a future which completes with an error. +/// - An exception is thrown to the zone handler from a timer task. +Future<Object?>? _runFailingTest(ClassMirror classMirror, Symbol symbol) { + var passed = false; + return runZonedGuarded(() { + // ignore: void_checks + return Future.sync(() => _runTest(classMirror, symbol)).then<void>((_) { + passed = true; + test_package.fail('Test passed - expected to fail.'); + }).catchError((Object e) { + // if passed, and we call fail(), rethrow this exception + if (passed) { + // ignore: only_throw_errors + throw e; + } + // otherwise, an exception is not a failure for _runFailingTest + }); + }, (e, st) { + // if passed, and we call fail(), rethrow this exception + if (passed) { + // ignore: only_throw_errors + throw e; + } + // otherwise, an exception is not a failure for _runFailingTest + }); +} + +Future<void> _runTest(ClassMirror classMirror, Symbol symbol) async { + var instanceMirror = classMirror.newInstance(const Symbol(''), []); + try { + await _invokeSymbolIfExists(instanceMirror, #setUp); + await instanceMirror.invoke(symbol, []).reflectee; + } finally { + await _invokeSymbolIfExists(instanceMirror, #tearDown); + } +} + +typedef _TestFunction = dynamic Function(); + +/// A marker annotation used to annotate test methods which are expected to +/// fail. +class FailingTest { + /// Initialize this annotation with the given arguments. + /// + /// [issue] is a full URI describing the failure and used for tracking. + /// [reason] is a free form textual description. + const FailingTest({String? issue, String? reason}); +} + +/// A marker annotation used to annotate test methods which are skipped. +class SkippedTest { + /// Initialize this annotation with the given arguments. + /// + /// [issue] is a full URI describing the failure and used for tracking. + /// [reason] is a free form textual description. + const SkippedTest({String? issue, String? reason}); +} + +/// A marker annotation used to annotate test methods with additional timeout +/// information. +class TestTimeout { + final test_package.Timeout _timeout; + + /// Initialize this annotation with the given timeout. + const TestTimeout(test_package.Timeout timeout) : _timeout = timeout; +} + +/// A marker annotation used to annotate test methods which are expected to fail +/// when asserts are enabled. +class _AssertFailingTest { + const _AssertFailingTest(); +} + +/// Information about a type based test group. +class _Group { + final bool isSolo; + final String name; + final List<_Test> tests = <_Test>[]; + + _Group(this.isSolo, this.name); + + bool get hasSoloTest => tests.any((test) => test.isSolo); + + void addSkippedTest(String name) { + var fullName = _combineNames(this.name, name); + tests.add(_Test.skipped(isSolo, fullName)); + } + + void addTest(bool isSolo, String name, MethodMirror memberMirror, + _TestFunction function) { + var fullName = _combineNames(this.name, name); + var timeout = + _getAnnotationInstance(memberMirror, TestTimeout) as TestTimeout?; + tests.add(_Test(isSolo, fullName, function, timeout?._timeout)); + } +} + +/// A marker annotation used to instruct dart2js to keep reflection information +/// for the annotated classes. +class _ReflectiveTest { + const _ReflectiveTest(); +} + +/// A marker annotation used to annotate "solo" groups and tests. +class _SoloTest { + const _SoloTest(); +} + +/// Information about a test. +class _Test { + final bool isSolo; + final String name; + final _TestFunction function; + final test_package.Timeout? timeout; + + final bool isSkipped; + + _Test(this.isSolo, this.name, this.function, this.timeout) + : isSkipped = false; + + _Test.skipped(this.isSolo, this.name) + : isSkipped = true, + function = (() {}), + timeout = null; +}
diff --git a/pkgs/test_reflective_loader/pubspec.yaml b/pkgs/test_reflective_loader/pubspec.yaml new file mode 100644 index 0000000..c30323c --- /dev/null +++ b/pkgs/test_reflective_loader/pubspec.yaml
@@ -0,0 +1,13 @@ +name: test_reflective_loader +version: 0.2.3-wip +description: Support for discovering tests and test suites using reflection. +repository: https://github.com/dart-lang/test_reflective_loader + +environment: + sdk: ^3.1.0 + +dependencies: + test: ^1.16.0 + +dev_dependencies: + dart_flutter_team_lints: ^2.0.0
diff --git a/pkgs/test_reflective_loader/test/test_reflective_loader_test.dart b/pkgs/test_reflective_loader/test/test_reflective_loader_test.dart new file mode 100644 index 0000000..fad98a5 --- /dev/null +++ b/pkgs/test_reflective_loader/test/test_reflective_loader_test.dart
@@ -0,0 +1,48 @@ +// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// ignore_for_file: non_constant_identifier_names + +import 'dart:async'; + +import 'package:test/test.dart'; +import 'package:test_reflective_loader/test_reflective_loader.dart'; + +void main() { + defineReflectiveSuite(() { + defineReflectiveTests(TestReflectiveLoaderTest); + }); +} + +@reflectiveTest +class TestReflectiveLoaderTest { + void test_passes() { + expect(true, true); + } + + @failingTest + void test_fails() { + expect(false, true); + } + + @failingTest + void test_fails_throws_sync() { + throw StateError('foo'); + } + + @failingTest + Future test_fails_throws_async() { + return Future.error('foo'); + } + + @skippedTest + void test_fails_but_skipped() { + throw StateError('foo'); + } + + @skippedTest + void test_times_out_but_skipped() { + while (true) {} + } +}