blob: 811e876664541ef3bb641ae8d3a0df1207b5ea3e [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io' as io;
import 'package:path/path.dart' as pathlib;
import 'package:web_driver_installer/chrome_driver_installer.dart';
import 'chrome_installer.dart';
import 'common.dart';
import 'environment.dart';
import 'utils.dart';
class IntegrationTestsManager {
final String _browser;
/// Installation directory for browser's driver.
/// Always re-install since driver can change frequently.
/// It usually changes with each the browser version changes.
/// A better solution would be installing the browser and the driver at the
/// same time.
// TODO(nurhan):
final io.Directory _browserDriverDir;
/// This is the parent directory for all drivers.
/// This directory is saved to [temporaryDirectories] and deleted before
/// tests shutdown.
final io.Directory _drivers;
: this._browserDriverDir = io.Directory(pathlib.join(
environment.webUiDartToolDir.path, 'drivers', _browser)),
this._drivers = io.Directory(
pathlib.join(environment.webUiDartToolDir.path, 'drivers'));
Future<bool> runTests() async {
if (_browser != 'chrome') {
print('WARNING: integration tests are only supported on chrome for now');
return false;
} else {
await prepareDriver();
// TODO(nurhan):
return await _runTests();
Future<bool> _runPubGet(String workingDirectory) async {
final String executable = isCirrus ? environment.pubExecutable : 'flutter';
final List<String> arguments = isCirrus
? <String>[
: <String>[
final int exitCode = await runProcess(
workingDirectory: workingDirectory,
if (exitCode != 0) {
'ERROR: Failed to run pub get. Exited with exit code $exitCode');
return false;
} else {
return true;
void _runDriver() async {
workingDirectory: io.Directory.current.path
print('INFO: Driver started');
void prepareDriver() async {
if (_browserDriverDir.existsSync()) {
_browserDriverDir.deleteSync(recursive: true);
_browserDriverDir.createSync(recursive: true);
io.Directory temp = io.Directory.current;
io.Directory.current = _browserDriverDir;
// TODO(nurhan):
final String chromeDriverVersion = await queryChromeDriverVersion();
ChromeDriverInstaller chromeDriverInstaller =
await chromeDriverInstaller.install(alwaysInstall: true);
await _runDriver();
io.Directory.current = temp;
/// Runs all the web tests under e2e_tests/web.
Future<bool> _runTests() async {
// Only list the files under e2e_tests/web.
final List<io.FileSystemEntity> entities =
environment.integrationTestsDir.listSync(followLinks: false);
bool allTestsPassed = true;
for (io.FileSystemEntity e in entities) {
// The tests should be under this directories.
if (e is io.Directory) {
allTestsPassed = allTestsPassed && await _validateAndRunTests(e);
return allTestsPassed;
/// Run tests in a single directory under: e2e_tests/web.
/// Run `flutter pub get` as the first step.
/// Validate the directory before running the tests. Each directory is
/// expected to be a test project which includes a `pubspec.yaml` file
/// and a `test_driver` directory.
Future<bool> _validateAndRunTests(io.Directory directory) async {
await _runPubGet(directory.path);
final bool testResults = await _runTestsInDirectory(directory);
return testResults;
Future<bool> _runTestsInDirectory(io.Directory directory) async {
final io.Directory testDirectory =
io.Directory(pathlib.join(directory.path, 'test_driver'));
final List<io.File> entities = testDirectory
.listSync(followLinks: false)
final List<String> e2eTestsToRun = List<String>();
// The following loops over the contents of the directory and saves an
// expected driver file name for each e2e test assuming any dart file
// not ending with `_test.dart` is an e2e test.
// Other files are not considered since developers can add files such as
for (io.File f in entities) {
final String basename = pathlib.basename(f.path);
if (!basename.contains('_test.dart') && basename.endsWith('.dart')) {
'INFO: In project ${directory} ${e2eTestsToRun.length} tests to run.');
int numberOfPassedTests = 0;
int numberOfFailedTests = 0;
for (String fileName in e2eTestsToRun) {
final bool testResults =
await _runTestsInProfileMode(directory, fileName);
if (testResults) {
} else {
final int numberOfTestsRun = numberOfPassedTests + numberOfFailedTests;
print('INFO: ${numberOfTestsRun} tests run. ${numberOfPassedTests} passed '
'and ${numberOfFailedTests} failed.');
return numberOfFailedTests == 0;
Future<bool> _runTestsInProfileMode(
io.Directory directory, String testName) async {
final int exitCode = await runProcess(
workingDirectory: directory.path,
if (exitCode != 0) {
final String statementToRun = 'flutter drive '
'--target=test_driver/${testName} -d web-server --profile '
'--browser-name=$_browser --local-engine=host_debug_unopt';
.writeln('ERROR: Failed to run test. Exited with exit code $exitCode'
'. Statement to run $testName locally use the following '
return false;
} else {
return true;
/// Validate the directory has a `pubspec.yaml` file and a `test_driver`
/// directory.
/// Also check the validity of files under `test_driver` directory calling
/// [_checkE2ETestsValidity] method.
void _validateTestDirectory(io.Directory directory) {
final List<io.FileSystemEntity> entities =
directory.listSync(followLinks: false);
// Whether the project has the pubspec.yaml file.
bool pubSpecFound = false;
// The test directory 'test_driver'.
io.Directory testDirectory = null;
for (io.FileSystemEntity e in entities) {
// The tests should be under this directories.
final String baseName = pathlib.basename(e.path);
if (e is io.Directory && baseName == 'test_driver') {
testDirectory = e;
if (e is io.File && baseName == 'pubspec.yaml') {
pubSpecFound = true;
if (!pubSpecFound) {
throw StateError('ERROR: pubspec.yaml file not found in the test project '
'in the directory ${directory.path}.');
if (testDirectory == null) {
throw StateError(
'ERROR: test_driver folder not found in the test project.'
'in the directory ${directory.path}.');
} else {
/// Checks if each e2e test file in the directory has a driver test
/// file to run it.
/// Prints informative message to the developer if an error has found.
/// For each e2e test which has name {name}.dart there will be a driver
/// file which drives it. The driver file should be named:
/// {name}_test.dart
void _checkE2ETestsValidity(io.Directory testDirectory) {
final Iterable<io.Directory> directories =
testDirectory.listSync(followLinks: false).whereType<io.Directory>();
if (directories.length > 0) {
throw StateError('${testDirectory.path} directory should not contain '
'any sub-directories');
final Iterable<io.File> entities =
testDirectory.listSync(followLinks: false).whereType<io.File>();
final Set<String> expectedDriverFileNames = Set<String>();
final Set<String> foundDriverFileNames = Set<String>();
int numberOfTests = 0;
// The following loops over the contents of the directory and saves an
// expected driver file name for each e2e test assuming any file
// not ending with `_test.dart` is an e2e test.
for (io.File f in entities) {
final String basename = pathlib.basename(f.path);
if (basename.contains('_test.dart')) {
// First remove this from expectedSet if not there add to the foundSet.
if (!expectedDriverFileNames.remove(basename)) {
} else if (basename.contains('.dart')) {
// Only run on dart files.
final String e2efileName = pathlib.basenameWithoutExtension(f.path);
final String expectedDriverName = '${e2efileName}_test.dart';
// First remove this from foundSet if not there add to the expectedSet.
if (!foundDriverFileNames.remove(expectedDriverName)) {
if (numberOfTests == 0) {
throw StateError(
'WARNING: No tests to run in this directory ${testDirectory.path}');
// TODO(nurhan): In order to reduce the work required from team members,
// remove the need for driver file, by using the same template file.
// Some driver files are missing.
if (expectedDriverFileNames.length > 0) {
for (String expectedDriverName in expectedDriverFileNames) {
print('ERROR: Test driver file named has ${expectedDriverName} '
'not found under directory ${testDirectory.path}. Stopping the '
'integration tests. Please add ${expectedDriverName}. Check to '
'README file on more details on how to setup integration tests.');
throw StateError('Error in test files. Check the logs for '
'further instructions');