blob: 083117458d448c02f5d2eff196bc50d219a64224 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
A top level harness to run all unit-tests in a specific engine build.
import argparse
import glob
import os
import re
import subprocess
import sys
import time
buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..'))
out_dir = os.path.join(buildroot_dir, 'out')
golden_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'resources')
fonts_dir = os.path.join(buildroot_dir, 'flutter', 'third_party', 'txt', 'third_party', 'fonts')
roboto_font_path = os.path.join(fonts_dir, 'Roboto-Regular.ttf')
dart_tests_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'dart',)
font_subset_dir = os.path.join(buildroot_dir, 'flutter', 'tools', 'font-subset')
fml_unittests_filter = '--gtest_filter=-*TimeSensitiveTest*'
def PrintDivider(char='='):
print '\n'
for _ in xrange(4):
print(''.join([char for _ in xrange(80)]))
print '\n'
def RunCmd(cmd, **kwargs):
command_string = ' '.join(cmd)
print 'Running command "%s"' % command_string
start_time = time.time()
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
(output, _) = process.communicate()
end_time = time.time()
# Print the result no matter what.
for line in output.splitlines():
print line
if process.returncode != 0:
raise Exception('Command "%s" exited with code %d' % (command_string, process.returncode))
print 'Command run successfully in %.2f seconds: %s' % (end_time - start_time, command_string)
def IsMac():
return sys.platform == 'darwin'
def IsLinux():
return sys.platform.startswith('linux')
def IsWindows():
return sys.platform.startswith(('cygwin', 'win'))
def ExecutableSuffix():
return '.exe' if IsWindows() else ''
def FindExecutablePath(path):
if os.path.exists(path):
return path
if IsWindows():
exe_path = path + '.exe'
if os.path.exists(exe_path):
return exe_path
bat_path = path + '.bat'
if os.path.exists(bat_path):
return bat_path
raise Exception('Executable %s does not exist!' % path)
def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildroot_dir):
if filter is not None and executable_name not in filter:
print('Skipping %s due to filter.' % executable_name)
executable = FindExecutablePath(os.path.join(build_dir, executable_name))
print('Running %s in %s' % (executable_name, cwd))
test_command = [ executable ] + flags
print(' '.join(test_command))
RunCmd(test_command, cwd=cwd)
def RunCCTests(build_dir, filter):
print("Running Engine Unit-tests.")
shuffle_flags = [
RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter, shuffle_flags)
RunEngineExecutable(build_dir, 'common_cpp_core_unittests', filter, shuffle_flags)
RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter, shuffle_flags)
if not IsWindows():
RunEngineExecutable(build_dir, 'embedder_unittests', filter, shuffle_flags)
RunEngineExecutable(build_dir, 'flutter_windows_unittests', filter, shuffle_flags)
RunEngineExecutable(build_dir, 'client_wrapper_windows_unittests', filter, shuffle_flags)
flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold']
if IsLinux():
flow_flags = [
'--golden-dir=%s' % golden_dir,
'--font-file=%s' % roboto_font_path,
RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags + shuffle_flags)
# TODO(44614): Re-enable after has been addressed.
# RunEngineExecutable(build_dir, 'fml_unittests', filter, [ fml_unittests_filter ] + shuffle_flags)
RunEngineExecutable(build_dir, 'runtime_unittests', filter, shuffle_flags)
if not IsWindows():
RunEngineExecutable(build_dir, 'shell_unittests', filter, shuffle_flags)
RunEngineExecutable(build_dir, 'ui_unittests', filter, shuffle_flags)
RunEngineExecutable(build_dir, 'testing_unittests', filter, shuffle_flags)
# These unit-tests are Objective-C and can only run on Darwin.
if IsMac():
RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter, shuffle_flags)
if IsLinux():
RunEngineExecutable(build_dir, 'txt_unittests', filter, shuffle_flags)
def RunEngineBenchmarks(build_dir, filter):
print("Running Engine Benchmarks.")
RunEngineExecutable(build_dir, 'shell_benchmarks', filter)
RunEngineExecutable(build_dir, 'fml_benchmarks', filter)
if IsLinux():
RunEngineExecutable(build_dir, 'txt_benchmarks', filter)
def SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot):
print("Generating snapshot for test %s" % dart_file)
dart = os.path.join(build_dir, 'dart-sdk', 'bin', 'dart')
frontend_server = os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot')
flutter_patched_sdk = os.path.join(build_dir, 'flutter_patched_sdk')
test_packages = os.path.join(dart_tests_dir, '.packages')
assert os.path.exists(dart)
assert os.path.exists(frontend_server)
assert os.path.exists(flutter_patched_sdk)
assert os.path.exists(test_packages)
snapshot_command = [
if verbose_dart_snapshot:
RunCmd(snapshot_command, cwd=buildroot_dir)
subprocess.check_output(snapshot_command, cwd=buildroot_dir)
assert os.path.exists(kernel_file_output)
def RunDartTest(build_dir, dart_file, verbose_dart_snapshot, multithreaded):
kernel_file_name = os.path.basename(dart_file) + '.kernel.dill'
kernel_file_output = os.path.join(out_dir, kernel_file_name)
SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot)
command_args = [
if multithreaded:
threading = 'multithreaded'
command_args.insert(0, '--force-multithreading')
threading = 'single-threaded'
print("Running test '%s' using 'flutter_tester' (%s)" % (kernel_file_name, threading))
RunEngineExecutable(build_dir, 'flutter_tester', None, command_args)
def RunPubGet(build_dir, directory):
print("Running 'pub get' in the tests directory %s" % dart_tests_dir)
pub_get_command = [
os.path.join(build_dir, 'dart-sdk', 'bin', 'pub'),
RunCmd(pub_get_command, cwd=directory)
def EnsureDebugUnoptSkyPackagesAreBuilt():
variant_out_dir = os.path.join(out_dir, 'host_debug_unopt')
ninja_command = [
# Attempt running Ninja if the out directory exists.
# We don't want to blow away any custom GN args the caller may have already set.
if os.path.exists(variant_out_dir):
RunCmd(ninja_command, cwd=buildroot_dir)
gn_command = [
os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
RunCmd(gn_command, cwd=buildroot_dir)
RunCmd(ninja_command, cwd=buildroot_dir)
def EnsureJavaTestsAreBuilt(android_out_dir):
ninja_command = [
# Attempt running Ninja if the out directory exists.
# We don't want to blow away any custom GN args the caller may have already set.
if os.path.exists(android_out_dir):
RunCmd(ninja_command, cwd=buildroot_dir)
# Otherwise prepare the directory first, then build the test.
gn_command = [
os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
RunCmd(gn_command, cwd=buildroot_dir)
RunCmd(ninja_command, cwd=buildroot_dir)
def AssertExpectedJavaVersion():
# `java -version` is output to stderr.
version_output = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT)
match = bool(re.compile('version "%s' % EXPECTED_VERSION).search(version_output))
message = "JUnit tests need to be run with Java %s. Check the `java -version` on your PATH." % EXPECTED_VERSION
assert match, message
def RunJavaTests(filter, android_variant='android_debug_unopt'):
android_out_dir = os.path.join(out_dir, android_variant)
embedding_deps_dir = os.path.join(buildroot_dir, 'third_party', 'android_embedding_dependencies', 'lib')
classpath = map(str, [
os.path.join(buildroot_dir, 'third_party', 'android_tools', 'sdk', 'platforms', 'android-29', 'android.jar'),
os.path.join(embedding_deps_dir, '*'), # Wildcard for all jars in the directory
os.path.join(android_out_dir, 'flutter.jar'),
os.path.join(android_out_dir, 'robolectric_tests.jar')
test_class = filter if filter else 'io.flutter.FlutterTestSuite'
command = [
'-Drobolectric.dependency.dir=' + embedding_deps_dir,
'-classpath', ':'.join(classpath),
def RunDartTests(build_dir, filter, verbose_dart_snapshot):
# This one is a bit messy. The pubspec.yaml at flutter/testing/dart/pubspec.yaml
# has dependencies that are hardcoded to point to the sky packages at host_debug_unopt/
# Before running Dart tests, make sure to run just that target (NOT the whole engine)
# Now that we have the Sky packages at the hardcoded location, run `pub get`.
RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'pub'), None, flags=['get'], cwd=dart_tests_dir)
dart_tests = glob.glob('%s/*.dart' % dart_tests_dir)
for dart_test_file in dart_tests:
if filter is not None and os.path.basename(dart_test_file) not in filter:
print("Skipping %s due to filter." % dart_test_file)
print("Testing dart file %s" % dart_test_file)
RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, True)
RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, False)
def RunFrontEndServerTests(build_dir):
test_dir = os.path.join(buildroot_dir, 'flutter', 'flutter_frontend_server')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
for dart_test_file in dart_tests:
opts = [
os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'),
os.path.join(build_dir, 'flutter_patched_sdk')]
os.path.join('dart-sdk', 'bin', 'dart'),
def RunConstFinderTests(build_dir):
test_dir = os.path.join(buildroot_dir, 'flutter', 'tools', 'const_finder', 'test')
opts = [
os.path.join(test_dir, 'const_finder_test.dart'),
os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'),
os.path.join(build_dir, 'flutter_patched_sdk')]
RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'dart'), None, flags=opts, cwd=test_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--variant', dest='variant', action='store',
default='host_debug_unopt', help='The engine build variant to run the tests for.');
parser.add_argument('--type', type=str, default='all')
parser.add_argument('--engine-filter', type=str, default='',
help='A list of engine test executables to run.')
parser.add_argument('--dart-filter', type=str, default='',
help='A list of Dart test scripts to run.')
parser.add_argument('--java-filter', type=str, default='',
help='A single Java test class to run.')
parser.add_argument('--android-variant', dest='android_variant', action='store',
help='The engine build variant to run java tests for')
parser.add_argument('--verbose-dart-snapshot', dest='verbose_dart_snapshot', action='store_true',
default=False, help='Show extra dart snapshot logging.')
args = parser.parse_args()
if args.type == 'all':
types = ['engine', 'dart', 'benchmarks', 'java', 'font-subset']
types = args.type.split(',')
build_dir = os.path.join(out_dir, args.variant)
if args.type != 'java':
assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir
engine_filter = args.engine_filter.split(',') if args.engine_filter else None
if 'engine' in types:
RunCCTests(build_dir, engine_filter)
if 'dart' in types:
assert not IsWindows(), "Dart tests can't be run on windows."
dart_filter = args.dart_filter.split(',') if args.dart_filter else None
RunDartTests(build_dir, dart_filter, args.verbose_dart_snapshot)
if 'java' in types:
assert not IsWindows(), "Android engine files can't be compiled on Windows."
java_filter = args.java_filter
if ',' in java_filter or '*' in java_filter:
print('Can only filter JUnit4 tests by single entire class name, eg "io.flutter.SmokeTest". Ignoring filter=' + java_filter)
java_filter = None
RunJavaTests(java_filter, args.android_variant)
if 'benchmarks' in types and not IsWindows():
RunEngineBenchmarks(build_dir, engine_filter)
if 'engine' in types or 'font-subset' in types:
RunCmd(['python', ''], cwd=font_subset_dir)
if __name__ == '__main__':