| #!/usr/bin/env python |
| # Copyright 2013 The Flutter Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """ |
| A top level harness to run all unit-tests in a specific engine build. |
| """ |
| |
| import argparse |
| import glob |
| import os |
| import re |
| import subprocess |
| import sys |
| import time |
| |
| buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..')) |
| out_dir = os.path.join(buildroot_dir, 'out') |
| golden_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'resources') |
| fonts_dir = os.path.join(buildroot_dir, 'flutter', 'third_party', 'txt', 'third_party', 'fonts') |
| roboto_font_path = os.path.join(fonts_dir, 'Roboto-Regular.ttf') |
| dart_tests_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'dart',) |
| font_subset_dir = os.path.join(buildroot_dir, 'flutter', 'tools', 'font-subset') |
| |
| fml_unittests_filter = '--gtest_filter=-*TimeSensitiveTest*' |
| |
| def PrintDivider(char='='): |
| print '\n' |
| for _ in xrange(4): |
| print(''.join([char for _ in xrange(80)])) |
| print '\n' |
| |
| def RunCmd(cmd, **kwargs): |
| command_string = ' '.join(cmd) |
| |
| PrintDivider('>') |
| print 'Running command "%s"' % command_string |
| |
| start_time = time.time() |
| process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) |
| (output, _) = process.communicate() |
| end_time = time.time() |
| |
| # Print the result no matter what. |
| for line in output.splitlines(): |
| print line |
| |
| if process.returncode != 0: |
| PrintDivider('!') |
| raise Exception('Command "%s" exited with code %d' % (command_string, process.returncode)) |
| |
| PrintDivider('<') |
| print 'Command run successfully in %.2f seconds: %s' % (end_time - start_time, command_string) |
| |
| |
| def IsMac(): |
| return sys.platform == 'darwin' |
| |
| |
| def IsLinux(): |
| return sys.platform.startswith('linux') |
| |
| |
| def IsWindows(): |
| return sys.platform.startswith(('cygwin', 'win')) |
| |
| |
| def ExecutableSuffix(): |
| return '.exe' if IsWindows() else '' |
| |
| def FindExecutablePath(path): |
| if os.path.exists(path): |
| return path |
| |
| if IsWindows(): |
| exe_path = path + '.exe' |
| if os.path.exists(exe_path): |
| return exe_path |
| |
| bat_path = path + '.bat' |
| if os.path.exists(bat_path): |
| return bat_path |
| |
| raise Exception('Executable %s does not exist!' % path) |
| |
| |
| def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildroot_dir): |
| if filter is not None and executable_name not in filter: |
| print('Skipping %s due to filter.' % executable_name) |
| return |
| |
| executable = FindExecutablePath(os.path.join(build_dir, executable_name)) |
| |
| print('Running %s in %s' % (executable_name, cwd)) |
| test_command = [ executable ] + flags |
| print(' '.join(test_command)) |
| RunCmd(test_command, cwd=cwd) |
| |
| |
| def RunCCTests(build_dir, filter): |
| print("Running Engine Unit-tests.") |
| |
| shuffle_flags = [ |
| "--gtest_shuffle", |
| "--gtest_repeat=2", |
| ] |
| |
| RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter, shuffle_flags) |
| |
| RunEngineExecutable(build_dir, 'common_cpp_core_unittests', filter, shuffle_flags) |
| |
| RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter, shuffle_flags) |
| |
| # https://github.com/flutter/flutter/issues/36294 |
| if not IsWindows(): |
| RunEngineExecutable(build_dir, 'embedder_unittests', filter, shuffle_flags) |
| else: |
| RunEngineExecutable(build_dir, 'flutter_windows_unittests', filter, shuffle_flags) |
| |
| RunEngineExecutable(build_dir, 'client_wrapper_windows_unittests', filter, shuffle_flags) |
| |
| flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold'] |
| if IsLinux(): |
| flow_flags = [ |
| '--golden-dir=%s' % golden_dir, |
| '--font-file=%s' % roboto_font_path, |
| ] |
| RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags + shuffle_flags) |
| |
| # TODO(44614): Re-enable after https://github.com/flutter/flutter/issues/44614 has been addressed. |
| # RunEngineExecutable(build_dir, 'fml_unittests', filter, [ fml_unittests_filter ] + shuffle_flags) |
| |
| RunEngineExecutable(build_dir, 'runtime_unittests', filter, shuffle_flags) |
| |
| # https://github.com/flutter/flutter/issues/36295 |
| if not IsWindows(): |
| RunEngineExecutable(build_dir, 'shell_unittests', filter, shuffle_flags) |
| |
| RunEngineExecutable(build_dir, 'ui_unittests', filter, shuffle_flags) |
| |
| RunEngineExecutable(build_dir, 'testing_unittests', filter, shuffle_flags) |
| |
| # These unit-tests are Objective-C and can only run on Darwin. |
| if IsMac(): |
| RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter, shuffle_flags) |
| |
| # https://github.com/flutter/flutter/issues/36296 |
| if IsLinux(): |
| RunEngineExecutable(build_dir, 'txt_unittests', filter, shuffle_flags) |
| |
| |
| def RunEngineBenchmarks(build_dir, filter): |
| print("Running Engine Benchmarks.") |
| |
| RunEngineExecutable(build_dir, 'shell_benchmarks', filter) |
| |
| RunEngineExecutable(build_dir, 'fml_benchmarks', filter) |
| |
| if IsLinux(): |
| RunEngineExecutable(build_dir, 'txt_benchmarks', filter) |
| |
| |
| |
| def SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot): |
| print("Generating snapshot for test %s" % dart_file) |
| |
| dart = os.path.join(build_dir, 'dart-sdk', 'bin', 'dart') |
| frontend_server = os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot') |
| flutter_patched_sdk = os.path.join(build_dir, 'flutter_patched_sdk') |
| test_packages = os.path.join(dart_tests_dir, '.packages') |
| |
| assert os.path.exists(dart) |
| assert os.path.exists(frontend_server) |
| assert os.path.exists(flutter_patched_sdk) |
| assert os.path.exists(test_packages) |
| |
| snapshot_command = [ |
| dart, |
| frontend_server, |
| '--sdk-root', |
| flutter_patched_sdk, |
| '--incremental', |
| '--target=flutter', |
| '--packages', |
| test_packages, |
| '--output-dill', |
| kernel_file_output, |
| dart_file |
| ] |
| |
| if verbose_dart_snapshot: |
| RunCmd(snapshot_command, cwd=buildroot_dir) |
| else: |
| subprocess.check_output(snapshot_command, cwd=buildroot_dir) |
| assert os.path.exists(kernel_file_output) |
| |
| |
| def RunDartTest(build_dir, dart_file, verbose_dart_snapshot, multithreaded): |
| kernel_file_name = os.path.basename(dart_file) + '.kernel.dill' |
| kernel_file_output = os.path.join(out_dir, kernel_file_name) |
| |
| SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot) |
| |
| command_args = [ |
| '--disable-observatory', |
| '--use-test-fonts', |
| kernel_file_output |
| ] |
| |
| if multithreaded: |
| threading = 'multithreaded' |
| command_args.insert(0, '--force-multithreading') |
| else: |
| threading = 'single-threaded' |
| |
| print("Running test '%s' using 'flutter_tester' (%s)" % (kernel_file_name, threading)) |
| RunEngineExecutable(build_dir, 'flutter_tester', None, command_args) |
| |
| def RunPubGet(build_dir, directory): |
| print("Running 'pub get' in the tests directory %s" % dart_tests_dir) |
| |
| pub_get_command = [ |
| os.path.join(build_dir, 'dart-sdk', 'bin', 'pub'), |
| 'get' |
| ] |
| RunCmd(pub_get_command, cwd=directory) |
| |
| |
| def EnsureDebugUnoptSkyPackagesAreBuilt(): |
| variant_out_dir = os.path.join(out_dir, 'host_debug_unopt') |
| |
| ninja_command = [ |
| 'autoninja', |
| '-C', |
| variant_out_dir, |
| 'flutter/sky/packages' |
| ] |
| |
| # Attempt running Ninja if the out directory exists. |
| # We don't want to blow away any custom GN args the caller may have already set. |
| if os.path.exists(variant_out_dir): |
| RunCmd(ninja_command, cwd=buildroot_dir) |
| return |
| |
| gn_command = [ |
| os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'), |
| '--runtime-mode', |
| 'debug', |
| '--unopt', |
| '--no-lto', |
| ] |
| |
| RunCmd(gn_command, cwd=buildroot_dir) |
| RunCmd(ninja_command, cwd=buildroot_dir) |
| |
| def EnsureJavaTestsAreBuilt(android_out_dir): |
| ninja_command = [ |
| 'autoninja', |
| '-C', |
| android_out_dir, |
| 'flutter/shell/platform/android:robolectric_tests' |
| ] |
| |
| # Attempt running Ninja if the out directory exists. |
| # We don't want to blow away any custom GN args the caller may have already set. |
| if os.path.exists(android_out_dir): |
| RunCmd(ninja_command, cwd=buildroot_dir) |
| return |
| |
| # Otherwise prepare the directory first, then build the test. |
| gn_command = [ |
| os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'), |
| '--android', |
| '--unoptimized', |
| '--runtime-mode=debug', |
| '--no-lto', |
| ] |
| RunCmd(gn_command, cwd=buildroot_dir) |
| RunCmd(ninja_command, cwd=buildroot_dir) |
| |
| def AssertExpectedJavaVersion(): |
| EXPECTED_VERSION = '1.8' |
| # `java -version` is output to stderr. https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4380614 |
| version_output = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) |
| match = bool(re.compile('version "%s' % EXPECTED_VERSION).search(version_output)) |
| message = "JUnit tests need to be run with Java %s. Check the `java -version` on your PATH." % EXPECTED_VERSION |
| assert match, message |
| |
| def RunJavaTests(filter, android_variant='android_debug_unopt'): |
| AssertExpectedJavaVersion() |
| android_out_dir = os.path.join(out_dir, android_variant) |
| EnsureJavaTestsAreBuilt(android_out_dir) |
| |
| embedding_deps_dir = os.path.join(buildroot_dir, 'third_party', 'android_embedding_dependencies', 'lib') |
| classpath = map(str, [ |
| os.path.join(buildroot_dir, 'third_party', 'android_tools', 'sdk', 'platforms', 'android-29', 'android.jar'), |
| os.path.join(embedding_deps_dir, '*'), # Wildcard for all jars in the directory |
| os.path.join(android_out_dir, 'flutter.jar'), |
| os.path.join(android_out_dir, 'robolectric_tests.jar') |
| ]) |
| |
| test_class = filter if filter else 'io.flutter.FlutterTestSuite' |
| command = [ |
| 'java', |
| '-Drobolectric.offline=true', |
| '-Drobolectric.dependency.dir=' + embedding_deps_dir, |
| '-classpath', ':'.join(classpath), |
| '-Drobolectric.logging=stdout', |
| 'org.junit.runner.JUnitCore', |
| test_class |
| ] |
| |
| RunCmd(command) |
| |
| def RunDartTests(build_dir, filter, verbose_dart_snapshot): |
| # This one is a bit messy. The pubspec.yaml at flutter/testing/dart/pubspec.yaml |
| # has dependencies that are hardcoded to point to the sky packages at host_debug_unopt/ |
| # Before running Dart tests, make sure to run just that target (NOT the whole engine) |
| EnsureDebugUnoptSkyPackagesAreBuilt(); |
| |
| # Now that we have the Sky packages at the hardcoded location, run `pub get`. |
| RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'pub'), None, flags=['get'], cwd=dart_tests_dir) |
| |
| dart_tests = glob.glob('%s/*.dart' % dart_tests_dir) |
| |
| for dart_test_file in dart_tests: |
| if filter is not None and os.path.basename(dart_test_file) not in filter: |
| print("Skipping %s due to filter." % dart_test_file) |
| else: |
| print("Testing dart file %s" % dart_test_file) |
| RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, True) |
| RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, False) |
| |
| |
| def RunFrontEndServerTests(build_dir): |
| test_dir = os.path.join(buildroot_dir, 'flutter', 'flutter_frontend_server') |
| dart_tests = glob.glob('%s/test/*_test.dart' % test_dir) |
| for dart_test_file in dart_tests: |
| opts = [ |
| dart_test_file, |
| os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'), |
| os.path.join(build_dir, 'flutter_patched_sdk')] |
| RunEngineExecutable( |
| build_dir, |
| os.path.join('dart-sdk', 'bin', 'dart'), |
| None, |
| flags=opts, |
| cwd=test_dir) |
| |
| |
| def RunConstFinderTests(build_dir): |
| test_dir = os.path.join(buildroot_dir, 'flutter', 'tools', 'const_finder', 'test') |
| opts = [ |
| os.path.join(test_dir, 'const_finder_test.dart'), |
| os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'), |
| os.path.join(build_dir, 'flutter_patched_sdk')] |
| RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'dart'), None, flags=opts, cwd=test_dir) |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| |
| parser.add_argument('--variant', dest='variant', action='store', |
| default='host_debug_unopt', help='The engine build variant to run the tests for.'); |
| parser.add_argument('--type', type=str, default='all') |
| parser.add_argument('--engine-filter', type=str, default='', |
| help='A list of engine test executables to run.') |
| parser.add_argument('--dart-filter', type=str, default='', |
| help='A list of Dart test scripts to run.') |
| parser.add_argument('--java-filter', type=str, default='', |
| help='A single Java test class to run.') |
| parser.add_argument('--android-variant', dest='android_variant', action='store', |
| default='android_debug_unopt', |
| help='The engine build variant to run java tests for') |
| parser.add_argument('--verbose-dart-snapshot', dest='verbose_dart_snapshot', action='store_true', |
| default=False, help='Show extra dart snapshot logging.') |
| |
| args = parser.parse_args() |
| |
| if args.type == 'all': |
| types = ['engine', 'dart', 'benchmarks', 'java', 'font-subset'] |
| else: |
| types = args.type.split(',') |
| |
| build_dir = os.path.join(out_dir, args.variant) |
| if args.type != 'java': |
| assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir |
| |
| engine_filter = args.engine_filter.split(',') if args.engine_filter else None |
| if 'engine' in types: |
| RunCCTests(build_dir, engine_filter) |
| |
| if 'dart' in types: |
| assert not IsWindows(), "Dart tests can't be run on windows. https://github.com/flutter/flutter/issues/36301." |
| dart_filter = args.dart_filter.split(',') if args.dart_filter else None |
| RunDartTests(build_dir, dart_filter, args.verbose_dart_snapshot) |
| RunConstFinderTests(build_dir) |
| RunFrontEndServerTests(build_dir) |
| |
| if 'java' in types: |
| assert not IsWindows(), "Android engine files can't be compiled on Windows." |
| java_filter = args.java_filter |
| if ',' in java_filter or '*' in java_filter: |
| print('Can only filter JUnit4 tests by single entire class name, eg "io.flutter.SmokeTest". Ignoring filter=' + java_filter) |
| java_filter = None |
| RunJavaTests(java_filter, args.android_variant) |
| |
| # https://github.com/flutter/flutter/issues/36300 |
| if 'benchmarks' in types and not IsWindows(): |
| RunEngineBenchmarks(build_dir, engine_filter) |
| |
| if 'engine' in types or 'font-subset' in types: |
| RunCmd(['python', 'test.py'], cwd=font_subset_dir) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |