pkgs/test/test/runner/configuration/randomize_order_test.dart

// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

@TestOn('vm')

import 'package:test_descriptor/test_descriptor.dart' as d;

import 'package:test/test.dart';

import '../../io.dart';

void main() {
  test('shuffles test order when passed a seed', () async {
    await d.file('test.dart', '''
      import 'package:test/test.dart';

      void main() {
        test("test 1", () {});
        test("test 2", () {});
        test("test 3", () {});
        test("test 4", () {});
      }
    ''').create();

    // Test with a given seed
    var test =
        await runTest(['test.dart', '--test-randomize-ordering-seed=987654']);
    expect(
        test.stdout,
        containsInOrder([
          '+0: test 4',
          '+1: test 3',
          '+2: test 1',
          '+3: test 2',
          '+4: All tests passed!'
        ]));
    await test.shouldExit(0);

    // Do not shuffle when passed 0
    test = await runTest(['test.dart', '--test-randomize-ordering-seed=0']);
    expect(
        test.stdout,
        containsInOrder([
          '+0: test 1',
          '+1: test 2',
          '+2: test 3',
          '+3: test 4',
          '+4: All tests passed!'
        ]));
    await test.shouldExit(0);

    // Do not shuffle when passed nothing
    test = await runTest(['test.dart']);
    expect(
        test.stdout,
        containsInOrder([
          '+0: test 1',
          '+1: test 2',
          '+2: test 3',
          '+3: test 4',
          '+4: All tests passed!'
        ]));
    await test.shouldExit(0);

    // Shuffle when passed random
    test =
        await runTest(['test.dart', '--test-randomize-ordering-seed=random']);
    expect(
        test.stdout,
        emitsInAnyOrder([
          contains('Shuffling test order with --test-randomize-ordering-seed'),
          isNot(contains(
              'Shuffling test order with --test-randomize-ordering-seed=0'))
        ]));
    await test.shouldExit(0);

    // Doesn't log about shuffling with the json reporter
    test = await runTest(
        ['test.dart', '--test-randomize-ordering-seed=random', '-r', 'json']);
    expect(test.stdout, neverEmits(contains('Shuffling test order')));
    await test.shouldExit(0);
  });

  test('shuffles each suite with the same seed', () async {
    await d.file('1_test.dart', '''
      import 'package:test/test.dart';

      void main() {
        test("test 1.1", () {});
        test("test 1.2", () {});
        test("test 1.3", () {});
      }
    ''').create();

    await d.file('2_test.dart', '''
      import 'package:test/test.dart';

      void main() {
        test("test 2.1", () {});
        test("test 2.2", () {});
        test("test 2.3", () {});
      }
    ''').create();

    var test = await runTest(['.', '--test-randomize-ordering-seed=12345']);
    expect(
        test.stdout,
        emitsInAnyOrder([
          containsInOrder([
            './1_test.dart: test 1.2',
            './1_test.dart: test 1.3',
            './1_test.dart: test 1.1'
          ]),
          containsInOrder([
            './2_test.dart: test 2.2',
            './2_test.dart: test 2.3',
            './2_test.dart: test 2.1'
          ]),
          contains('+6: All tests passed!')
        ]));
    await test.shouldExit(0);
  });

  test('shuffles groups as well as tests in groups', () async {
    await d.file('test.dart', '''
      import 'package:test/test.dart';

      void main() {
      group("Group 1", () {
        test("test 1.1", () {});
        test("test 1.2", () {});
        test("test 1.3", () {});
        test("test 1.4", () {});
       });
      group("Group 2", () {
        test("test 2.1", () {});
        test("test 2.2", () {});
        test("test 2.3", () {});
        test("test 2.4", () {});
       }); 
      }
    ''').create();

    // Test with a given seed
    var test =
        await runTest(['test.dart', '--test-randomize-ordering-seed=123']);
    expect(
        test.stdout,
        containsInOrder([
          '+0: Group 2 test 2.4',
          '+1: Group 2 test 2.2',
          '+2: Group 2 test 2.1',
          '+3: Group 2 test 2.3',
          '+4: Group 1 test 1.4',
          '+5: Group 1 test 1.2',
          '+6: Group 1 test 1.1',
          '+7: Group 1 test 1.3',
          '+8: All tests passed!'
        ]));
    await test.shouldExit(0);
  });

  test('shuffles nested groups', () async {
    await d.file('test.dart', '''
      import 'package:test/test.dart';

      void main() {
      group("Group 1", () {
        test("test 1.1", () {});
        test("test 1.2", () {});
        group("Group 2", () {
          test("test 2.3", () {});
          test("test 2.4", () {});
        });
       });
      }
    ''').create();

    var test =
        await runTest(['test.dart', '--test-randomize-ordering-seed=123']);
    expect(
        test.stdout,
        containsInOrder([
          '+0: Group 1 test 1.1',
          '+1: Group 1 Group 2 test 2.4',
          '+2: Group 1 Group 2 test 2.3',
          '+3: Group 1 test 1.2',
          '+4: All tests passed!'
        ]));
    await test.shouldExit(0);
  });
}