pkg/unittest/lib/unittest.dart

// Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/**
 * A library for writing dart unit tests.
 *
 * ## Installing ##
 *
 * Use [pub][] to install this package. Add the following to your `pubspec.yaml`
 * file.
 *
 *     dependencies:
 *       unittest: any
 *
 * Then run `pub install`.
 *
 * For more information, see the
 * [unittest package on pub.dartlang.org][pkg].
 *
 * See the [Getting Started](http://pub.dartlang.org/doc)
 * guide for more details.
 *
 * ##Concepts##
 *
 *  * __Tests__: Tests are specified via the top-level function [test], they can be
 *    organized together using [group].
 *  * __Checks__: Test expectations can be specified via [expect]
 *  * __Matchers__: [expect] assertions are written declaratively using the
 *    [Matcher] class.
 *  * __Configuration__: The framework can be adapted by calling [configure] with a
 *    [Configuration]. See the other libraries in the `unittest` package for
 *    alternative implementations of [Configuration] including
 *    `compact_vm_config.dart`, `html_config.dart` and
 *    `html_enhanced_config.dart`.
 *
 * ##Examples##
 *
 * A trivial test:
 *
 *     import 'package:unittest/unittest.dart';
 *     main() {
 *       test('this is a test', () {
 *         int x = 2 + 3;
 *         expect(x, equals(5));
 *       });
 *     }
 *
 * Multiple tests:
 *
 *     import 'package:unittest/unittest.dart';
 *     main() {
 *       test('this is a test', () {
 *         int x = 2 + 3;
 *         expect(x, equals(5));
 *       });
 *       test('this is another test', () {
 *         int x = 2 + 3;
 *         expect(x, equals(5));
 *       });
 *     }
 *
 * Multiple tests, grouped by category:
 *
 *     import 'package:unittest/unittest.dart';
 *     main() {
 *       group('group A', () {
 *         test('test A.1', () {
 *           int x = 2 + 3;
 *           expect(x, equals(5));
 *         });
 *         test('test A.2', () {
 *           int x = 2 + 3;
 *           expect(x, equals(5));
 *         });
 *       });
 *       group('group B', () {
 *         test('this B.1', () {
 *           int x = 2 + 3;
 *           expect(x, equals(5));
 *         });
 *       });
 *     }
 *
 * Asynchronous tests: if callbacks expect between 0 and 2 positional arguments,
 * depending on the suffix of expectAsyncX(). expectAsyncX() will wrap a
 * function into a new callback and will not consider the test complete until
 * that callback is run. A count argument can be provided to specify the number
 * of times the callback should be called (the default is 1).
 *
 *     import 'package:unittest/unittest.dart';
 *     import 'dart:isolate';
 *     main() {
 *       test('callback is executed once', () {
 *         // wrap the callback of an asynchronous call with [expectAsync0] if
 *         // the callback takes 0 arguments...
 *         var timer = Timer.run(expectAsync0(() {
 *           int x = 2 + 3;
 *           expect(x, equals(5));
 *         }));
 *       });
 *
 *       test('callback is executed twice', () {
 *         var callback = expectAsync0(() {
 *           int x = 2 + 3;
 *           expect(x, equals(5));
 *         }, count: 2); // <-- we can indicate multiplicity to [expectAsync0]
 *         Timer.run(callback);
 *         Timer.run(callback);
 *       });
 *     }
 *
 * expectAsyncX() will wrap the callback code in a try/catch handler to handle
 * exceptions (treated as test failures). There may be times when the number of
 * times a callback should be called is non-deterministic. In this case a dummy
 * callback can be created with expectAsync0((){}) and this can be called from
 * the real callback when it is finally complete. In this case the body of the
 * callback should be protected within a call to guardAsync(); this will ensure
 * that exceptions are properly handled.
 *
 * A variation on this is expectAsyncUntilX(), which takes a callback as the
 * first parameter and a predicate function as the second parameter; after each
 * time * the callback is called, the predicate function will be called; if it
 * returns false the test will still be considered incomplete.
 *
 * Test functions can return [Future]s, which provide another way of doing
 * asynchronous tests. The test framework will handle exceptions thrown by
 * the Future, and will advance to the next test when the Future is complete.
 * It is still important to use expectAsync/guardAsync with any parts of the
 * test that may be invoked from a top level context (for example, with
 * Timer.run()], as the Future exception handler may not capture exceptions
 * in such code.
 *
 * Note: due to some language limitations we have to use different functions
 * depending on the number of positional arguments of the callback. In the
 * future, we plan to expose a single `expectAsync` function that can be used
 * regardless of the number of positional arguments. This requires new langauge
 * features or fixes to the current spec (e.g. see
 * [Issue 2706](http://dartbug.com/2706)).
 *
 * Meanwhile, we plan to add this alternative API for callbacks of more than 2
 * arguments or that take named parameters. (this is not implemented yet,
 * but will be coming here soon).
 *
 *     import 'package:unittest/unittest.dart';
 *     import 'dart:isolate';
 *     main() {
 *       test('callback is executed', () {
 *         // indicate ahead of time that an async callback is expected.
 *         var async = startAsync();
 *         Timer.run(() {
 *           // Guard the body of the callback, so errors are propagated
 *           // correctly.
 *           guardAsync(() {
 *             int x = 2 + 3;
 *             expect(x, equals(5));
 *           });
 *           // indicate that the asynchronous callback was invoked.
 *           async.complete();
 *         });
 *       });
 *     }
 *
 * [pub]: http://pub.dartlang.org
 * [pkg]: http://pub.dartlang.org/packages/unittest
 */
library unittest;

import 'dart:async';
import 'dart:isolate';
import 'dart:collection';
import 'matcher.dart';
export 'matcher.dart';

// TODO(amouravski): We should not need to import mock here, but it's necessary
// to enable dartdoc on the mock library, as it's not picked up normally.
import 'mock.dart';

part 'src/config.dart';
part 'src/test_case.dart';

Configuration _config;

/** [Configuration] used by the unittest library. */
Configuration get unittestConfiguration => _config;

/**
 * Sets the [Configuration] used by the unittest library.
 *
 * Throws a [StateError] if there is an existing, incompatible value.
 */
void set unittestConfiguration(Configuration value) {
  if(!identical(_config, value)) {
    if(_config != null) {
      throw new StateError('unittestConfiguration has already been set');
    }
    _config = value;
  }
}

/**
 * Can be called by tests to log status. Tests should use this
 * instead of [print].
 */
void logMessage(String message) =>
    _config.onLogMessage(currentTestCase, message);

/** Separator used between group names and test names. */
String groupSep = ' ';

final List<TestCase> _testCases = new List<TestCase>();

/** Tests executed in this suite. */
final List<TestCase> testCases = new UnmodifiableListView<TestCase>(_testCases);

/**
 * Setup and teardown functions for a group and its parents, the latter
 * for chaining.
 */
class GroupContext {
  final GroupContext parent;

  /** Description text of the current test group. */
  final String _name;

  /** Setup function called before each test in a group. */
  Function _testSetup;

  get testSetup => _testSetup;

  get parentSetup => (parent == null) ? null : parent.testSetup;

  set testSetup(Function setup) {
    var preSetup = parentSetup;
    if (preSetup == null) {
      _testSetup = setup;
    } else {
      _testSetup = () {
        var f = preSetup();
        if (f is Future) {
          return f.then((_) => setup());
        } else {
          return setup();
        }
      };
    }
  }

  /** Teardown function called after each test in a group. */
  Function _testTeardown;

  get testTeardown => _testTeardown;

  get parentTeardown => (parent == null) ? null : parent.testTeardown;

  set testTeardown(Function teardown) {
    var postTeardown = parentTeardown;
    if (postTeardown == null) {
      _testTeardown = teardown;
    } else {
      _testTeardown = () {
        var f = teardown();
        if (f is Future) {
          return f.then((_) => postTeardown());
        } else {
          return postTeardown();
        }
      };
    }
  }

  String get fullName => (parent == null || parent == _rootContext)
      ? _name
      : "${parent.fullName}$groupSep$_name";

  GroupContext([this.parent, this._name = '']) {
    _testSetup = parentSetup;
    _testTeardown = parentTeardown;
  }
}

// We use a 'dummy' context for the top level to eliminate null
// checks when querying the context. This allows us to easily
//  support top-level setUp/tearDown functions as well.
GroupContext _rootContext = new GroupContext();
GroupContext _currentContext = _rootContext;

int _currentTestCaseIndex = 0;

/** [TestCase] currently being executed. */
TestCase get currentTestCase =>
    (_currentTestCaseIndex >= 0 && _currentTestCaseIndex < testCases.length)
        ? testCases[_currentTestCaseIndex]
        : null;

/** Whether the framework is in an initialized state. */
bool _initialized = false;

String _uncaughtErrorMessage = null;

/** Test case result strings. */
// TODO(gram) we should change these constants to use a different string
// (so that writing 'FAIL' in the middle of a test doesn't
// imply that the test fails). We can't do it without also changing
// the testrunner and test.dart though.
const PASS  = 'pass';
const FAIL  = 'fail';
const ERROR = 'error';

/** If set, then all other test cases will be ignored. */
TestCase _soloTest;

/**
 * A map that can be used to communicate state between a test driver
 * or main() function and the tests, particularly when these two
 * are otherwise independent. For example, a test driver that starts
 * an HTTP server and then runs tests that access that server could use
 * this as a way of communicating the server port to the tests.
 */
Map testState = {};

/**
 * Creates a new test case with the given description and body. The
 * description will include the descriptions of any surrounding group()
 * calls.
 */
void test(String spec, TestFunction body) {
  ensureInitialized();
  _testCases.add(new TestCase._internal(testCases.length + 1, _fullSpec(spec),
                                        body));
}

/**
 * Creates a new test case with the given description and body. The
 * description will include the descriptions of any surrounding group()
 * calls.
 *
 * "solo_" means that this will be the only test that is run. All other tests
 * will be skipped. This is a convenience function to let you quickly isolate
 * a single test by adding "solo_" before it to temporarily disable all other
 * tests.
 */
void solo_test(String spec, TestFunction body) {
  // TODO(rnystrom): Support multiple solos. If more than one test is solo-ed,
  // all of the solo-ed tests and none of the non-solo-ed ones should run.
  if (_soloTest != null) {
    throw new Exception('Only one test can be soloed right now.');
  }

  ensureInitialized();

  _soloTest = new TestCase._internal(testCases.length + 1, _fullSpec(spec),
                                     body);
  _testCases.add(_soloTest);
}

/** Sentinel value for [_SpreadArgsHelper]. */
class _Sentinel {
  const _Sentinel();
}

/** Simulates spread arguments using named arguments. */
// TODO(sigmund): remove this class and simply use a closure with named
// arguments (if still applicable).
class _SpreadArgsHelper {
  final Function callback;
  final int minExpectedCalls;
  final int maxExpectedCalls;
  final Function isDone;
  final String id;
  int actualCalls = 0;
  final TestCase testCase;
  bool complete;
  static const sentinel = const _Sentinel();

  _SpreadArgsHelper(Function callback, int minExpected, int maxExpected,
      Function isDone, String id)
      : this.callback = callback,
        minExpectedCalls = minExpected,
        maxExpectedCalls = (maxExpected == 0 && minExpected > 0)
            ? minExpected
            : maxExpected,
        this.isDone = isDone,
        this.testCase = currentTestCase,
        this.id = _makeCallbackId(id, callback) {
    ensureInitialized();
    if (testCase == null) {
      throw new StateError("No valid test. Did you forget to run your test "
          "inside a call to test()?");
    }

    if (isDone != null || minExpected > 0) {
      testCase._callbackFunctionsOutstanding++;
      complete = false;
    } else {
      complete = true;
    }
  }

  static String _makeCallbackId(String id, Function callback) {
    // Try to create a reasonable id.
    if (id != null) {
      return "$id ";
    } else {
      // If the callback is not an anonymous closure, try to get the
      // name.
      var fname = callback.toString();
      var prefix = "Function '";
      var pos = fname.indexOf(prefix);
      if (pos > 0) {
        pos += prefix.length;
        var epos = fname.indexOf("'", pos);
        if (epos > 0) {
          return "${fname.substring(pos, epos)} ";
        }
      }
    }
    return '';
  }

  bool shouldCallBack() {
    ++actualCalls;
    if (testCase.isComplete) {
      // Don't run if the test is done. We don't throw here as this is not
      // the current test, but we do mark the old test as having an error
      // if it previously passed.
      if (testCase.result == PASS) {
        testCase.error(
            'Callback ${id}called ($actualCalls) after test case '
            '${testCase.description} has already been marked as '
            '${testCase.result}.', '');
      }
      return false;
    } else if (maxExpectedCalls >= 0 && actualCalls > maxExpectedCalls) {
      throw new TestFailure('Callback ${id}called more times than expected '
                            '($maxExpectedCalls).');
    }
    return true;
  }

  void after() {
    if (!complete) {
      if (minExpectedCalls > 0 && actualCalls < minExpectedCalls) return;
      if (isDone != null && !isDone()) return;

      // Mark this callback as complete and remove it from the testcase
      // oustanding callback count; if that hits zero the testcase is done.
      complete = true;
      testCase._markCallbackComplete();
    }
  }

  invoke0() {
    return _guardAsync(
        () {
          if (shouldCallBack()) {
            return callback();
          }
        },
        after, testCase);
  }

  invoke1(arg1) {
    return _guardAsync(
        () {
          if (shouldCallBack()) {
            return callback(arg1);
          }
        },
        after, testCase);
  }

  invoke2(arg1, arg2) {
    return _guardAsync(
        () {
          if (shouldCallBack()) {
            return callback(arg1, arg2);
          }
        },
        after, testCase);
  }
}

/**
 * Indicate that [callback] is expected to be called a [count] number of times
 * (by default 1). The unittest framework will wait for the callback to run the
 * specified [count] times before it continues with the following test.  Using
 * [expectAsync0] will also ensure that errors that occur within [callback] are
 * tracked and reported. [callback] should take 0 positional arguments (named
 * arguments are not supported). [id] can be used to provide more
 * descriptive error messages if the callback is called more often than
 * expected. [max] can be used to specify an upper bound on the number of
 * calls; if this is exceeded the test will fail (or be marked as in error if
 * it was already complete). A value of 0 for [max] (the default) will set
 * the upper bound to the same value as [count]; i.e. the callback should be
 * called exactly [count] times. A value of -1 for [max] will mean no upper
 * bound.
 */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function expectAsync0(Function callback,
                     {int count: 1, int max: 0, String id}) {
  return new _SpreadArgsHelper(callback, count, max, null, id).invoke0;
}

/** Like [expectAsync0] but [callback] should take 1 positional argument. */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function expectAsync1(Function callback,
                     {int count: 1, int max: 0, String id}) {
  return new _SpreadArgsHelper(callback, count, max, null, id).invoke1;
}

/** Like [expectAsync0] but [callback] should take 2 positional arguments. */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function expectAsync2(Function callback,
                     {int count: 1, int max: 0, String id}) {
  return new _SpreadArgsHelper(callback, count, max, null, id).invoke2;
}

/**
 * Indicate that [callback] is expected to be called until [isDone] returns
 * true. The unittest framework check [isDone] after each callback and only
 * when it returns true will it continue with the following test. Using
 * [expectAsyncUntil0] will also ensure that errors that occur within
 * [callback] are tracked and reported. [callback] should take 0 positional
 * arguments (named arguments are not supported). [id] can be used to
 * identify the callback in error messages (for example if it is called
 * after the test case is complete).
 */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function expectAsyncUntil0(Function callback, Function isDone, {String id}) {
  return new _SpreadArgsHelper(callback, 0, -1, isDone, id).invoke0;
}

/**
 * Like [expectAsyncUntil0] but [callback] should take 1 positional argument.
 */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function expectAsyncUntil1(Function callback, Function isDone, {String id}) {
  return new _SpreadArgsHelper(callback, 0, -1, isDone, id).invoke1;
}

/**
 * Like [expectAsyncUntil0] but [callback] should take 2 positional arguments.
 */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function expectAsyncUntil2(Function callback, Function isDone, {String id}) {
  return new _SpreadArgsHelper(callback, 0, -1, isDone, id).invoke2;
}

/**
 * Wraps the [callback] in a new function and returns that function. The new
 * function will be able to handle exceptions by directing them to the correct
 * test. This is thus similar to expectAsync0. Use it to wrap any callbacks that
 * might optionally be called but may never be called during the test.
 * [callback] should take 0 positional arguments (named arguments are not
 * supported). [id] can be used to identify the callback in error
 * messages (for example if it is called after the test case is complete).
 */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function protectAsync0(Function callback, {String id}) {
  return new _SpreadArgsHelper(callback, 0, -1, null, id).invoke0;
}

/**
 * Like [protectAsync0] but [callback] should take 1 positional argument.
 */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function protectAsync1(Function callback, {String id}) {
  return new _SpreadArgsHelper(callback, 0, -1, null, id).invoke1;
}

/**
 * Like [protectAsync0] but [callback] should take 2 positional arguments.
 */
// TODO(sigmund): deprecate this API when issue 2706 is fixed.
Function protectAsync2(Function callback, {String id}) {
  return new _SpreadArgsHelper(callback, 0, -1, null, id).invoke2;
}

/**
 * Creates a new named group of tests. Calls to group() or test() within the
 * body of the function passed to this will inherit this group's description.
 */
void group(String description, void body()) {
  ensureInitialized();
  _currentContext = new GroupContext(_currentContext, description);
  try {
    body();
  } catch (e, trace) {
    var stack = (trace == null) ? '' : ': ${trace.toString()}';
    _uncaughtErrorMessage = "${e.toString()}$stack";
  } finally {
    // Now that the group is over, restore the previous one.
    _currentContext = _currentContext.parent;
  }
}

/**
 * Register a [setUp] function for a test [group]. This function will
 * be called before each test in the group is run.
 * [setUp] and [tearDown] should be called within the [group] before any
 * calls to [test]. The [setupTest] function can be asynchronous; in this
 * case it must return a [Future].
 */
void setUp(Function setupTest) {
  _currentContext.testSetup = setupTest;
}

/**
 * Register a [tearDown] function for a test [group]. This function will
 * be called after each test in the group is run. Note that if groups
 * are nested only the most locally scoped [teardownTest] function will be run.
 * [setUp] and [tearDown] should be called within the [group] before any
 * calls to [test]. The [teardownTest] function can be asynchronous; in this
 * case it must return a [Future].
 */
void tearDown(Function teardownTest) {
  _currentContext.testTeardown = teardownTest;
}

/** Advance to the next test case. */
void _nextTestCase() {
  runAsync(() {
    _currentTestCaseIndex++;
    _nextBatch();
  });
}

/**
 * Utility function that can be used to notify the test framework that an
 *  error was caught outside of this library.
 */
void _reportTestError(String msg, String trace) {
 if (_currentTestCaseIndex < testCases.length) {
    final testCase = testCases[_currentTestCaseIndex];
    testCase.error(msg, trace);
  } else {
    _uncaughtErrorMessage = "$msg: $trace";
  }
}

void rerunTests() {
  _uncaughtErrorMessage = null;
  _initialized = true; // We don't want to reset the test array.
  runTests();
}

/**
 * Filter the tests. [testFilter] can be a [RegExp], a [String] or a
 * predicate function. This is different to enabling/disabling tests
 * in that it removes the tests completely.
 */
void filterTests(testFilter) {
  var filterFunction;
  if (testFilter is String) {
    RegExp re = new RegExp(testFilter);
    filterFunction = (t) => re.hasMatch(t.description);
  } else if (testFilter is RegExp) {
    filterFunction = (t) => testFilter.hasMatch(t.description);
  } else if (testFilter is Function) {
    filterFunction = testFilter;
  }
  _testCases.retainWhere(filterFunction);
}

/** Runs all queued tests, one at a time. */
void runTests() {
  _ensureInitialized(false);
  _currentTestCaseIndex = 0;

  // If we are soloing a test, remove all the others.
  if (_soloTest != null) {
    filterTests((t) => t == _soloTest);
  }

  _config.onStart();

  runAsync(() {
    _nextBatch();
  });
}

/**
 * Run [tryBody] guarded in a try-catch block. If an exception is thrown, it is
 * passed to the corresponding test.
 *
 * The value returned by [tryBody] (if any) is returned by [guardAsync].
 */
guardAsync(Function tryBody) {
  return _guardAsync(tryBody, null, currentTestCase);
}

_guardAsync(Function tryBody, Function finallyBody, TestCase testCase) {
  assert(testCase != null);
  try {
    return tryBody();
  } catch (e, trace) {
    _registerException(testCase, e, trace);
  } finally {
    if (finallyBody != null) finallyBody();
  }
}

/**
 * Registers that an exception was caught for the current test.
 */
void registerException(e, [trace]) {
  _registerException(currentTestCase, e, trace);
}

/**
 * Registers that an exception was caught for the current test.
 */
void _registerException(TestCase testCase, e, [trace]) {
  trace = trace == null ? '' : trace.toString();
  String message = (e is TestFailure) ? e.message : 'Caught $e';
  if (testCase.result == null) {
    testCase.fail(message, trace);
  } else {
    testCase.error(message, trace);
  }
}

/**
 * Runs a batch of tests, yielding whenever an asynchronous test starts
 * running. Tests will resume executing when such asynchronous test calls
 * [done] or if it fails with an exception.
 */
void _nextBatch() {
  while (true) {
    if (_currentTestCaseIndex >= testCases.length) {
      _completeTests();
      break;
    }
    final testCase = testCases[_currentTestCaseIndex];
    var f = _guardAsync(testCase._run, null, testCase);
    if (f != null) {
      f.whenComplete(() {
        _nextTestCase(); // Schedule the next test.
      });
      break;
    }
    _currentTestCaseIndex++;
  }
}

/** Publish results on the page and notify controller. */
void _completeTests() {
  if (!_initialized) return;
  int passed = 0;
  int failed = 0;
  int errors = 0;

  for (TestCase t in testCases) {
    switch (t.result) {
      case PASS:  passed++; break;
      case FAIL:  failed++; break;
      case ERROR: errors++; break;
    }
  }
  _config.onSummary(passed, failed, errors, testCases, _uncaughtErrorMessage);
  _config.onDone(passed > 0 && failed == 0 && errors == 0 &&
      _uncaughtErrorMessage == null);
  _initialized = false;
}

String _fullSpec(String spec) {
  var group = '${_currentContext.fullName}';
  if (spec == null) return group;
  return group != '' ? '$group$groupSep$spec' : spec;
}

/**
 * Lazily initializes the test library if not already initialized.
 */
void ensureInitialized() {
  _ensureInitialized(true);
}

void _ensureInitialized(bool configAutoStart) {
  if (_initialized) {
    return;
  }
  _initialized = true;
  // Hook our async guard into the matcher library.
  wrapAsync = (f, [id]) => expectAsync1(f, id: id);

  _uncaughtErrorMessage = null;

  if (_config == null) {
    unittestConfiguration = new Configuration();
  }
  _config.onInit();

  if (configAutoStart && _config.autoStart) {
    // Immediately queue the suite up. It will run after a timeout (i.e. after
    // main() has returned).
    runAsync(runTests);
  }
}

/** Select a solo test by ID. */
void setSoloTest(int id) {
  for (var i = 0; i < testCases.length; i++) {
    if (testCases[i].id == id) {
      _soloTest = testCases[i];
      break;
    }
  }
}

/** Enable/disable a test by ID. */
void _setTestEnabledState(int testId, bool state) {
  // Try fast path first.
  if (testCases.length > testId && testCases[testId].id == testId) {
    testCases[testId].enabled = state;
  } else {
    for (var i = 0; i < testCases.length; i++) {
      if (testCases[i].id == testId) {
        testCases[i].enabled = state;
        break;
      }
    }
  }
}

/** Enable a test by ID. */
void enableTest(int testId) => _setTestEnabledState(testId, true);

/** Disable a test by ID. */
void disableTest(int testId) => _setTestEnabledState(testId, false);

/** Signature for a test function. */
typedef dynamic TestFunction();