test/runner/configuration/top_level_test.dart

// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

@TestOn("vm")

import 'dart:async';
import 'dart:convert';

import 'package:path/path.dart' as p;
import 'package:test_descriptor/test_descriptor.dart' as d;

import 'package:test/src/util/io.dart';
import 'package:test/test.dart';

import '../../io.dart';

void main() {
  test("ignores an empty file", () async {
    await d.file("dart_test.yaml", "").create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("success", () {});
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains("+1: All tests passed!")));
    await test.shouldExit(0);
  });

  test("loads configuration from the path passed to --configuration", () async {
    // Make sure dart_test.yaml is ignored.
    await d.file("dart_test.yaml", jsonEncode({"run_skipped": true})).create();

    await d.file("special_test.yaml", jsonEncode({"skip": true})).create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("test", () => throw "oh no");
      }
    """).create();

    var test =
        await runTest(["--configuration", "special_test.yaml", "test.dart"]);
    expect(test.stdout, emitsThrough(contains('All tests skipped.')));
    await test.shouldExit(0);
  });

  test("pauses the test runner after a suite loads with pause_after_load: true",
      () async {
    await d
        .file("dart_test.yaml", jsonEncode({"pause_after_load": true}))
        .create();

    await d.file("test.dart", """
import 'package:test/test.dart';

void main() {
  print('loaded test!');

  test("success", () {});
}
""").create();

    var test = await runTest(["-p", "chrome", "test.dart"]);
    await expectLater(test.stdout, emitsThrough("loaded test!"));
    await expectLater(
        test.stdout,
        emitsInOrder([
          "",
          equalsIgnoringWhitespace("""
            The test runner is paused. Open the dev console in Chrome and set
            breakpoints. Once you're finished, return to this terminal and press
            Enter.
          """)
        ]));

    var nextLineFired = false;
    test.stdout.next.then(expectAsync1((line) {
      expect(line, contains("+0: success"));
      nextLineFired = true;
    }));

    // Wait a little bit to be sure that the tests don't start running without
    // our input.
    await new Future.delayed(new Duration(seconds: 2));
    expect(nextLineFired, isFalse);

    test.stdin.writeln();
    await expectLater(
        test.stdout, emitsThrough(contains("+1: All tests passed!")));
    await test.shouldExit(0);
  }, tags: 'chrome');

  test("runs skipped tests with run_skipped: true", () async {
    await d.file("dart_test.yaml", jsonEncode({"run_skipped": true})).create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("skip", () => print("In test!"), skip: true);
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains("In test!")));
    expect(test.stdout, emitsThrough(contains("+1: All tests passed!")));
    await test.shouldExit(0);
  });

  test("includes the full stack with verbose_trace: true", () async {
    await d
        .file("dart_test.yaml", jsonEncode({"verbose_trace": true}))
        .create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("failure", () => throw "oh no");
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains("dart:isolate-patch")));
    await test.shouldExit(1);
  });

  test("disables stack trace chaining with chain_stack_traces: false",
      () async {
    await d
        .file("dart_test.yaml", jsonEncode({"chain_stack_traces": false}))
        .create();

    await d.file("test.dart", """
         import 'dart:async';

         import 'package:test/test.dart';

          void main() {
            test("failure", () async{
              await new Future((){});
              await new Future((){});
              throw "oh no";
            });
          }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(
        test.stdout,
        containsInOrder([
          "+0: failure",
          "+0 -1: failure [E]",
          "oh no",
          "test.dart 9:15  main.<fn>",
        ]));
    await test.shouldExit(1);
  });

  test("doesn't dartify stack traces for JS-compiled tests with js_trace: true",
      () async {
    await d.file("dart_test.yaml", jsonEncode({"js_trace": true})).create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("failure", () => throw "oh no");
      }
    """).create();

    var test = await runTest(["-p", "chrome", "--verbose-trace", "test.dart"]);
    expect(test.stdoutStream(), neverEmits(endsWith(" main.<fn>")));
    expect(test.stdoutStream(), neverEmits(contains("package:test")));
    expect(test.stdoutStream(), neverEmits(contains("dart:async/zone.dart")));
    expect(test.stdout, emitsThrough(contains("-1: Some tests failed.")));
    await test.shouldExit(1);
  }, tags: 'chrome');

  test("retries tests with retry: 1", () async {
    await d.file("dart_test.yaml", jsonEncode({"retry": 1})).create();

    await d.file("test.dart", """
      import 'package:test/test.dart';
      import 'dart:async';

      var attempt = 0;
      void main() {
        test("test", () {
          attempt++;
          if(attempt <= 1) {
            throw 'Failure!';
          }
        });
      }

    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains('+1: All tests passed')));
    await test.shouldExit(0);
  });

  test("skips tests with skip: true", () async {
    await d.file("dart_test.yaml", jsonEncode({"skip": true})).create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("test", () {});
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains('All tests skipped.')));
    await test.shouldExit(0);
  });

  test("skips tests with skip: reason", () async {
    await d
        .file("dart_test.yaml", jsonEncode({"skip": "Tests are boring."}))
        .create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("test", () {});
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains('Tests are boring.')));
    expect(test.stdout, emitsThrough(contains('All tests skipped.')));
    await test.shouldExit(0);
  });

  group("test_on", () {
    test("runs tests on a platform matching platform", () async {
      await d.file("dart_test.yaml", jsonEncode({"test_on": "vm"})).create();

      await d.file("test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("test", () {});
        }
      """).create();

      var test = await runTest(["test.dart"]);
      expect(test.stdout, emitsThrough(contains('All tests passed!')));
      await test.shouldExit(0);
    });

    test("warns about the VM when no OSes are supported", () async {
      await d
          .file("dart_test.yaml", jsonEncode({"test_on": "chrome"}))
          .create();

      await d.file("test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("test", () {});
        }
      """).create();

      var test = await runTest(["test.dart"]);
      expect(
          test.stderr,
          emits(
              "Warning: this package doesn't support running tests on the Dart "
              "VM."));
      expect(test.stdout, emitsThrough(contains('No tests ran.')));
      await test.shouldExit(0);
    });

    test("warns about the OS when some OSes are supported", () async {
      await d.file("dart_test.yaml", jsonEncode({"test_on": otherOS})).create();

      await d.file("test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("test", () {});
        }
      """).create();

      var test = await runTest(["test.dart"]);
      expect(
          test.stderr,
          emits("Warning: this package doesn't support running tests on "
              "${currentOS.name}."));
      expect(test.stdout, emitsThrough(contains('No tests ran.')));
      await test.shouldExit(0);
    });

    test("warns about browsers in general when no browsers are supported",
        () async {
      await d.file("dart_test.yaml", jsonEncode({"test_on": "vm"})).create();

      await d.file("test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("test", () {});
        }
      """).create();

      var test = await runTest(["-p", "chrome", "test.dart"]);
      expect(
          test.stderr,
          emits(
              "Warning: this package doesn't support running tests on browsers."));
      expect(test.stdout, emitsThrough(contains('No tests ran.')));
      await test.shouldExit(0);
    });

    test(
        "warns about specific browsers when specific browsers are "
        "supported", () async {
      await d
          .file("dart_test.yaml", jsonEncode({"test_on": "safari"}))
          .create();

      await d.file("test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("test", () {});
        }
      """).create();

      var test = await runTest(["-p", "chrome,firefox,phantomjs", "test.dart"]);
      expect(
          test.stderr,
          emits(
              "Warning: this package doesn't support running tests on Chrome, "
              "Firefox, or PhantomJS."));
      expect(test.stdout, emitsThrough(contains('No tests ran.')));
      await test.shouldExit(0);
    });
  });

  test("uses the specified reporter", () async {
    await d.file("dart_test.yaml", jsonEncode({"reporter": "json"})).create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("success", () {});
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains('"testStart"')));
    await test.shouldExit(0);
  });

  test("uses the specified pub serve port", () async {
    await d.file("pubspec.yaml", """
name: myapp
dependencies:
  barback: any
  test: {path: ${p.current}}
transformers:
- myapp:
    \$include: test/**_test.dart
- test/pub_serve:
    \$include: test/**_test.dart
""").create();

    await d.dir("lib", [
      d.file("myapp.dart", """
        import 'package:barback/barback.dart';

        class MyTransformer extends Transformer {
          final allowedExtensions = '.dart';

          MyTransformer.asPlugin();

          Future apply(Transform transform) async {
            var contents = await transform.primaryInput.readAsString();
            transform.addOutput(new Asset.fromString(
                transform.primaryInput.id,
                contents.replaceAll("isFalse", "isTrue")));
          }
        }
      """)
    ]).create();

    await (await runPub(['get'])).shouldExit(0);

    await d.dir("test", [
      d.file("my_test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("success", () => expect(true, isFalse));
        }
      """)
    ]).create();

    var pub = await runPubServe();

    await d
        .file("dart_test.yaml", jsonEncode({"pub_serve": pubServePort}))
        .create();

    var test = await runTest([]);
    expect(test.stdout, emitsThrough(contains('+1: All tests passed!')));
    await test.shouldExit(0);
    await pub.kill();
  }, tags: 'pub', skip: 'https://github.com/dart-lang/test/issues/821');

  test("uses the specified concurrency", () async {
    await d.file("dart_test.yaml", jsonEncode({"concurrency": 2})).create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("success", () {});
      }
    """).create();

    // We can't reliably test the concurrency, but this at least ensures that
    // it doesn't fail to parse.
    var test = await runTest(["test.dart"]);
    expect(test.stdout, emitsThrough(contains("+1: All tests passed!")));
    await test.shouldExit(0);
  });

  test("uses the specified timeout", () async {
    await d.file("dart_test.yaml", jsonEncode({"timeout": "0s"})).create();

    await d.file("test.dart", """
      import 'dart:async';

      import 'package:test/test.dart';

      void main() {
        test("success", () => new Future.delayed(Duration.zero));
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(
        test.stdout,
        containsInOrder(
            ["Test timed out after 0 seconds.", "-1: Some tests failed."]));
    await test.shouldExit(1);
  });

  test("runs on the specified platforms", () async {
    await d
        .file(
            "dart_test.yaml",
            jsonEncode({
              "platforms": ["vm", "chrome"]
            }))
        .create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("success", () {});
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, containsInOrder(["[VM] success", "[Chrome] success"]));
    await test.shouldExit(0);
  }, tags: "chrome");

  test("command line args take precedence", () async {
    await d.file("dart_test.yaml", jsonEncode({"timeout": "0s"})).create();

    await d.file("test.dart", """
      import 'dart:async';

      import 'package:test/test.dart';

      void main() {
        test("success", () => new Future.delayed(Duration.zero));
      }
    """).create();

    var test = await runTest(["--timeout=none", "test.dart"]);
    expect(test.stdout, emitsThrough(contains("All tests passed!")));
    await test.shouldExit(0);
  });

  test("uses the specified regexp names", () async {
    await d
        .file(
            "dart_test.yaml",
            jsonEncode({
              "names": ["z[ia]p", "a"]
            }))
        .create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("zip", () {});
        test("zap", () {});
        test("zop", () {});
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, containsInOrder(["+0: zap", "+1: All tests passed!"]));
    await test.shouldExit(0);
  });

  test("uses the specified plain names", () async {
    await d
        .file(
            "dart_test.yaml",
            jsonEncode({
              "names": ["z", "a"]
            }))
        .create();

    await d.file("test.dart", """
      import 'package:test/test.dart';

      void main() {
        test("zip", () {});
        test("zap", () {});
        test("zop", () {});
      }
    """).create();

    var test = await runTest(["test.dart"]);
    expect(test.stdout, containsInOrder(["+0: zap", "+1: All tests passed!"]));
    await test.shouldExit(0);
  });

  test("uses the specified paths", () async {
    await d
        .file(
            "dart_test.yaml",
            jsonEncode({
              "paths": ["zip", "zap"]
            }))
        .create();

    await d.dir("zip", [
      d.file("zip_test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("success", () {});
        }
      """)
    ]).create();

    await d.dir("zap", [
      d.file("zip_test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("success", () {});
        }
      """)
    ]).create();

    await d.dir("zop", [
      d.file("zip_test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("failure", () => throw "oh no");
        }
      """)
    ]).create();

    var test = await runTest([]);
    expect(test.stdout, emitsThrough(contains('All tests passed!')));
    await test.shouldExit(0);
  });

  test("uses the specified filename", () async {
    await d
        .file("dart_test.yaml", jsonEncode({"filename": "test_*.dart"}))
        .create();

    await d.dir("test", [
      d.file("test_foo.dart", """
        import 'package:test/test.dart';

        void main() {
          test("success", () {});
        }
      """),
      d.file("foo_test.dart", """
        import 'package:test/test.dart';

        void main() {
          test("failure", () => throw "oh no");
        }
      """),
      d.file("test_foo.bart", """
        import 'package:test/test.dart';

        void main() {
          test("failure", () => throw "oh no");
        }
      """)
    ]).create();

    var test = await runTest([]);
    expect(test.stdout, emitsThrough(contains('All tests passed!')));
    await test.shouldExit(0);
  });
}