pkg/test_runner/lib/src/test_configurations.dart - sdk.git - Git at Google

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.

 import 'dart:async';
 import 'dart:io';
 import 'dart:math' as math;

 import 'android.dart';
 import 'browser_controller.dart';
 import 'co19_test_config.dart';
 import 'configuration.dart';
 import 'fuchsia.dart';
 import 'path.dart';
 import 'process_queue.dart';
 import 'service/web_driver_service.dart';
 import 'terminal.dart';
 import 'test_progress.dart';
 import 'test_suite.dart';
 import 'utils.dart';

 export 'configuration.dart' show TestConfiguration;

 /// The directories that contain test suites which follow the conventions
 /// required by [StandardTestSuite]'s forDirectory constructor.
 ///
 /// New test suites should follow this convention because it makes it much
 /// simpler to add them to test.dart. Existing test suites should be moved to
 /// here, if possible.
 final testSuiteDirectories = [
   SuiteDirectory('pkg'),
   SuiteDirectory('runtime/tests/vm'),
   SuiteDirectory('samples'),
   SuiteDirectory('tests/corelib'),
   SuiteDirectory('tests/dartdevc'),
   SuiteDirectory('tests/ffi'),
   SuiteDirectory('tests/language'),
   SuiteDirectory('tests/macro_build'),
   SuiteDirectory('tests/lib'),
   SuiteDirectory('tests/standalone'),
   SuiteDirectory('tests/web'),
   SuiteDirectory('third_party/pkg/dart_style', 'test'),
   SuiteDirectory('third_party/pkg/dartdoc'),
   SuiteDirectory('third_party/pkg/native/pkgs/code_assets'),
   SuiteDirectory('third_party/pkg/native/pkgs/data_assets'),
   SuiteDirectory('third_party/pkg/native/pkgs/hooks_runner'),
   SuiteDirectory('third_party/pkg/native/pkgs/hooks'),
   SuiteDirectory('third_party/pkg/native/pkgs/native_toolchain_c'),
   SuiteDirectory('third_party/pkg/native/pkgs/record_use'),
   SuiteDirectory('third_party/pkg/package_config'),
   SuiteDirectory('utils/tests/peg'),
 ];

 /// Describes a directory whose name defines a test suite and whose contents
 /// are tests.
 final class SuiteDirectory {
   final Path directory;

   /// The subdirectory to look inside [directory] for tests.
   ///
   /// If `null`, then all tests in [directory] are found.
   final String? testSubdirectory;

   SuiteDirectory(String directoryPath, [this.testSubdirectory])
     : directory = Path(directoryPath);
 }

 // TODO(26372): Ensure that the returned future awaits on all started tasks.
 Future testConfigurations(List<TestConfiguration> configurations) async {
   var startTime = DateTime.now();

   // Extract global options from first configuration.
   var firstConf = configurations[0];
   var maxProcesses = firstConf.taskCount;
   var progress = firstConf.progress;
   var verbose = firstConf.isVerbose;
   var printTiming = firstConf.printTiming;
   var listTests = firstConf.listTests;
   var listStatusFiles = firstConf.listStatusFiles;
   var reportInJson = firstConf.reportInJson;

   Browser.resetBrowserConfiguration = firstConf.resetBrowser;
   DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null);

   // Print the configurations being run by this execution of
   // test.dart. However, don't do it if the silent progress indicator
   // is used.
   if (progress != Progress.silent) {
     Terminal.print(
       'Test configuration${configurations.length > 1 ? 's' : ''}:',
     );
     for (var configuration in configurations) {
       Terminal.print("    ${configuration.configuration}");
       Terminal.print(
         "Suites tested: ${configuration.selectors.keys.join(", ")}",
       );
     }
   }

   var runningBrowserTests = configurations.any(
     (config) => config.runtime.isBrowser,
   );

   var eventListeners = <EventListener>[];
   var testSuites = <TestSuite>[];
   var serverFutures = <Future>[];
   var maxBrowserProcesses = maxProcesses;
   if (configurations.length > 1 &&
       (configurations[0].testServerPort != 0 ||
           configurations[0].testServerCrossOriginPort != 0)) {
     Terminal.print(
       "If the http server ports are specified, only one configuration"
       " may be run at a time",
     );
     exit(1);
   }

   var services = <Future<WebDriverService>>{};
   for (var configuration in configurations) {
     if (!listTests && !listStatusFiles && runningBrowserTests) {
       serverFutures.add(configuration.startServers());
       if (WebDriverService.supportedRuntimes.contains(configuration.runtime)) {
         services.add(
           WebDriverService.startServiceForRuntime(configuration.runtime),
         );
       }
     }

     if (configuration.runtime.isSafari) {
       // Safari does not allow us to run from a fresh profile, so we can only
       // use one browser. Additionally, you can not start two simulators
       // for mobile safari simultaneously.
       maxBrowserProcesses = 1;
     } else if (configuration.runtime == Runtime.chrome &&
         Platform.operatingSystem == 'macos') {
       // Chrome on mac results in random timeouts.
       // Issue: https://github.com/dart-lang/sdk/issues/23891
       // This change does not fix the problem.
       maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2);
     } else if (configuration.runtime == Runtime.chromeOnAndroid) {
       maxBrowserProcesses = math.min(
         maxBrowserProcesses,
         (await AdbHelper.listDevices()).length,
       );
     }

     // If we specifically pass in a suite only run that.
     if (configuration.suiteDirectory != null) {
       var suitePath = Path(configuration.suiteDirectory!);
       testSuites.add(PackageTestSuite(configuration, suitePath));
     } else {
       for (var testSuiteDir in testSuiteDirectories) {
         var name = testSuiteDir.directory.filename;
         if (configuration.selectors.containsKey(name)) {
           testSuites.add(
             StandardTestSuite.forDirectory(
               configuration,
               testSuiteDir.directory,
               testSuiteDir.testSubdirectory,
             ),
           );
         }
       }

       for (var key in configuration.selectors.keys) {
         if (key == 'co19') {
           testSuites.add(Co19TestSuite(configuration, key));
         } else if ((configuration.compiler == Compiler.dartk ||
                 configuration.compiler == Compiler.dart2bytecode) &&
             configuration.runtime == Runtime.vm &&
             configuration.system != System.android &&
             configuration.system != System.fuchsia &&
             key == 'vm') {
           // vm tests contain both cc tests (added here) and dart tests (added
           // in [TEST_SUITE_DIRECTORIES]).
           // TODO(https://github.com/dart-lang/sdk/issues/63723): Support
           // Android and Fuchsia.
           testSuites.add(VMTestSuite(configuration));
         } else if (key == 'ffi_unit') {
           // 'ffi_unit' contains cc non-DartVM unit tests.
           //
           // This is a separate suite from 'ffi', because we want to run the
           // 'ffi' suite on many architectures, but 'ffi_unit' only on one.
           testSuites.add(FfiTestSuite(configuration));
         } else if (configuration.compiler == Compiler.dart2analyzer) {
           if (key == 'analyze_library') {
             testSuites.add(AnalyzeLibraryTestSuite(configuration));
           }
         }
       }
     }

     if (configuration.system == System.fuchsia) {
       await FuchsiaEmulator.instance().publishPackage(
         configuration.buildDirectory,
         configuration.mode.name,
         configuration.architecture.name,
       );
     }
     if (configuration.system == System.android &&
         (configuration.architecture == Architecture.x64 ||
             configuration.architecture == Architecture.x64c)) {
       await AndroidEmulators.start();
     }
   }

   for (var service in services) {
     serverFutures.add(service);
     service.then(eventListeners.add);
   }

   // If we only need to print out status files for test suites
   // we return from running here and just print.
   if (firstConf.listStatusFiles) {
     for (var suite in testSuites) {
       Terminal.print(suite.suiteName);
       for (var statusFile in suite.statusFilePaths.toSet()) {
         Terminal.print("\t$statusFile");
       }
     }
     return;
   }

   void allTestsFinished() {
     for (var configuration in configurations) {
       configuration.stopServers();
     }
     if (configurations.any((configuration) {
       return configuration.system == System.fuchsia;
     })) {
       FuchsiaEmulator.instance().stop(firstConf.isVerbose);
     }
     if (configurations.any((configuration) {
       return configuration.system == System.android;
     })) {
       AndroidEmulators.stop();
     }

     DebugLogger.close();
     if (!firstConf.keepGeneratedFiles) {
       TestUtils.deleteTempSnapshotDirectory(configurations[0]);
     }
   }

   // We don't print progress if we list tests.
   if (progress != Progress.silent && !listTests) {
     var formatter = Formatter.normal;
     if (progress == Progress.color) {
       progress = Progress.compact;
       formatter = Formatter.color;
     }

     eventListeners.add(SummaryPrinter());
     if (!firstConf.silentFailures) {
       eventListeners.add(TestFailurePrinter(formatter));
     }

     if (firstConf.printPassingStdout) {
       eventListeners.add(PassingStdoutPrinter(formatter));
     }

     var indicator = ProgressIndicator.fromProgress(
       progress,
       startTime,
       formatter,
     );
     if (indicator != null) eventListeners.add(indicator);

     if (printTiming) {
       eventListeners.add(TimingPrinter(startTime));
     }

     eventListeners.add(SkippedCompilationsPrinter());

     if (progress == Progress.status) {
       eventListeners.add(TimedProgressPrinter());
     }

     if (firstConf.reportFailures) {
       eventListeners.add(FailedTestsPrinter());
     }

     eventListeners.add(ResultCountPrinter(formatter));
   }

   if (firstConf.writeResults) {
     eventListeners.add(ResultWriter(firstConf.outputDirectory));
   }

   if (firstConf.copyCoreDumps) {
     eventListeners.add(UnexpectedCrashLogger());
   }

   // The only progress indicator when listing tests should be the
   // the summary printer.
   if (listTests) {
     eventListeners.add(SummaryPrinter(jsonOnly: reportInJson));
   } else {
     if (!firstConf.cleanExit) {
       eventListeners.add(ExitCodeSetter());
     }
     eventListeners.add(IgnoredTestMonitor());
   }

   // If any of the configurations need to access android devices we'll first
   // make a pool of all available adb devices.
   AdbDevicePool? adbDevicePool;
   var needsAdbDevicePool = configurations.any((conf) {
     return conf.system == System.android;
   });
   if (needsAdbDevicePool) {
     adbDevicePool = await AdbDevicePool.create();
   }

   // Start all the HTTP servers required before starting the process queue.
   if (serverFutures.isNotEmpty) {
     await Future.wait(serverFutures);
   }

   // [firstConf] is needed here, because the ProcessQueue uses some settings.
   ProcessQueue(
     firstConf,
     maxProcesses,
     maxBrowserProcesses,
     testSuites,
     eventListeners,
     allTestsFinished,
     verbose,
     adbDevicePool,
   );
 }
	// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
	// for details. All rights reserved. Use of this source code is governed by a
	// BSD-style license that can be found in the LICENSE file.

	import 'dart:async';
	import 'dart:io';
	import 'dart:math' as math;

	import 'android.dart';
	import 'browser_controller.dart';
	import 'co19_test_config.dart';
	import 'configuration.dart';
	import 'fuchsia.dart';
	import 'path.dart';
	import 'process_queue.dart';
	import 'service/web_driver_service.dart';
	import 'terminal.dart';
	import 'test_progress.dart';
	import 'test_suite.dart';
	import 'utils.dart';

	export 'configuration.dart' show TestConfiguration;

	/// The directories that contain test suites which follow the conventions
	/// required by [StandardTestSuite]'s forDirectory constructor.
	///
	/// New test suites should follow this convention because it makes it much
	/// simpler to add them to test.dart. Existing test suites should be moved to
	/// here, if possible.
	final testSuiteDirectories = [
	SuiteDirectory('pkg'),
	SuiteDirectory('runtime/tests/vm'),
	SuiteDirectory('samples'),
	SuiteDirectory('tests/corelib'),
	SuiteDirectory('tests/dartdevc'),
	SuiteDirectory('tests/ffi'),
	SuiteDirectory('tests/language'),
	SuiteDirectory('tests/macro_build'),
	SuiteDirectory('tests/lib'),
	SuiteDirectory('tests/standalone'),
	SuiteDirectory('tests/web'),
	SuiteDirectory('third_party/pkg/dart_style', 'test'),
	SuiteDirectory('third_party/pkg/dartdoc'),
	SuiteDirectory('third_party/pkg/native/pkgs/code_assets'),
	SuiteDirectory('third_party/pkg/native/pkgs/data_assets'),
	SuiteDirectory('third_party/pkg/native/pkgs/hooks_runner'),
	SuiteDirectory('third_party/pkg/native/pkgs/hooks'),
	SuiteDirectory('third_party/pkg/native/pkgs/native_toolchain_c'),
	SuiteDirectory('third_party/pkg/native/pkgs/record_use'),
	SuiteDirectory('third_party/pkg/package_config'),
	SuiteDirectory('utils/tests/peg'),
	];

	/// Describes a directory whose name defines a test suite and whose contents
	/// are tests.
	final class SuiteDirectory {
	final Path directory;

	/// The subdirectory to look inside [directory] for tests.
	///
	/// If `null`, then all tests in [directory] are found.
	final String? testSubdirectory;

	SuiteDirectory(String directoryPath, [this.testSubdirectory])
	: directory = Path(directoryPath);
	}

	// TODO(26372): Ensure that the returned future awaits on all started tasks.
	Future testConfigurations(List<TestConfiguration> configurations) async {
	var startTime = DateTime.now();

	// Extract global options from first configuration.
	var firstConf = configurations[0];
	var maxProcesses = firstConf.taskCount;
	var progress = firstConf.progress;
	var verbose = firstConf.isVerbose;
	var printTiming = firstConf.printTiming;
	var listTests = firstConf.listTests;
	var listStatusFiles = firstConf.listStatusFiles;
	var reportInJson = firstConf.reportInJson;

	Browser.resetBrowserConfiguration = firstConf.resetBrowser;
	DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null);

	// Print the configurations being run by this execution of
	// test.dart. However, don't do it if the silent progress indicator
	// is used.
	if (progress != Progress.silent) {
	Terminal.print(
	'Test configuration${configurations.length > 1 ? 's' : ''}:',
	);
	for (var configuration in configurations) {
	Terminal.print(" ${configuration.configuration}");
	Terminal.print(
	"Suites tested: ${configuration.selectors.keys.join(", ")}",
	);
	}
	}

	var runningBrowserTests = configurations.any(
	(config) => config.runtime.isBrowser,
	);

	var eventListeners = <EventListener>[];
	var testSuites = <TestSuite>[];
	var serverFutures = <Future>[];
	var maxBrowserProcesses = maxProcesses;
	if (configurations.length > 1 &&
	(configurations[0].testServerPort != 0 \|\|
	configurations[0].testServerCrossOriginPort != 0)) {
	Terminal.print(
	"If the http server ports are specified, only one configuration"
	" may be run at a time",
	);
	exit(1);
	}

	var services = <Future<WebDriverService>>{};
	for (var configuration in configurations) {
	if (!listTests && !listStatusFiles && runningBrowserTests) {
	serverFutures.add(configuration.startServers());
	if (WebDriverService.supportedRuntimes.contains(configuration.runtime)) {
	services.add(
	WebDriverService.startServiceForRuntime(configuration.runtime),
	);
	}
	}

	if (configuration.runtime.isSafari) {
	// Safari does not allow us to run from a fresh profile, so we can only
	// use one browser. Additionally, you can not start two simulators
	// for mobile safari simultaneously.
	maxBrowserProcesses = 1;
	} else if (configuration.runtime == Runtime.chrome &&
	Platform.operatingSystem == 'macos') {
	// Chrome on mac results in random timeouts.
	// Issue: https://github.com/dart-lang/sdk/issues/23891
	// This change does not fix the problem.
	maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2);
	} else if (configuration.runtime == Runtime.chromeOnAndroid) {
	maxBrowserProcesses = math.min(
	maxBrowserProcesses,
	(await AdbHelper.listDevices()).length,
	);
	}

	// If we specifically pass in a suite only run that.
	if (configuration.suiteDirectory != null) {
	var suitePath = Path(configuration.suiteDirectory!);
	testSuites.add(PackageTestSuite(configuration, suitePath));
	} else {
	for (var testSuiteDir in testSuiteDirectories) {
	var name = testSuiteDir.directory.filename;
	if (configuration.selectors.containsKey(name)) {
	testSuites.add(
	StandardTestSuite.forDirectory(
	configuration,
	testSuiteDir.directory,
	testSuiteDir.testSubdirectory,
	),
	);
	}
	}

	for (var key in configuration.selectors.keys) {
	if (key == 'co19') {
	testSuites.add(Co19TestSuite(configuration, key));
	} else if ((configuration.compiler == Compiler.dartk \|\|
	configuration.compiler == Compiler.dart2bytecode) &&
	configuration.runtime == Runtime.vm &&
	configuration.system != System.android &&
	configuration.system != System.fuchsia &&
	key == 'vm') {
	// vm tests contain both cc tests (added here) and dart tests (added
	// in [TEST_SUITE_DIRECTORIES]).
	// TODO(https://github.com/dart-lang/sdk/issues/63723): Support
	// Android and Fuchsia.
	testSuites.add(VMTestSuite(configuration));
	} else if (key == 'ffi_unit') {
	// 'ffi_unit' contains cc non-DartVM unit tests.
	//
	// This is a separate suite from 'ffi', because we want to run the
	// 'ffi' suite on many architectures, but 'ffi_unit' only on one.
	testSuites.add(FfiTestSuite(configuration));
	} else if (configuration.compiler == Compiler.dart2analyzer) {
	if (key == 'analyze_library') {
	testSuites.add(AnalyzeLibraryTestSuite(configuration));
	}
	}
	}
	}

	if (configuration.system == System.fuchsia) {
	await FuchsiaEmulator.instance().publishPackage(
	configuration.buildDirectory,
	configuration.mode.name,
	configuration.architecture.name,
	);
	}
	if (configuration.system == System.android &&
	(configuration.architecture == Architecture.x64 \|\|
	configuration.architecture == Architecture.x64c)) {
	await AndroidEmulators.start();
	}
	}

	for (var service in services) {
	serverFutures.add(service);
	service.then(eventListeners.add);
	}

	// If we only need to print out status files for test suites
	// we return from running here and just print.
	if (firstConf.listStatusFiles) {
	for (var suite in testSuites) {
	Terminal.print(suite.suiteName);
	for (var statusFile in suite.statusFilePaths.toSet()) {
	Terminal.print("\t$statusFile");
	}
	}
	return;
	}

	void allTestsFinished() {
	for (var configuration in configurations) {
	configuration.stopServers();
	}
	if (configurations.any((configuration) {
	return configuration.system == System.fuchsia;
	})) {
	FuchsiaEmulator.instance().stop(firstConf.isVerbose);
	}
	if (configurations.any((configuration) {
	return configuration.system == System.android;
	})) {
	AndroidEmulators.stop();
	}

	DebugLogger.close();
	if (!firstConf.keepGeneratedFiles) {
	TestUtils.deleteTempSnapshotDirectory(configurations[0]);
	}
	}

	// We don't print progress if we list tests.
	if (progress != Progress.silent && !listTests) {
	var formatter = Formatter.normal;
	if (progress == Progress.color) {
	progress = Progress.compact;
	formatter = Formatter.color;
	}

	eventListeners.add(SummaryPrinter());
	if (!firstConf.silentFailures) {
	eventListeners.add(TestFailurePrinter(formatter));
	}

	if (firstConf.printPassingStdout) {
	eventListeners.add(PassingStdoutPrinter(formatter));
	}

	var indicator = ProgressIndicator.fromProgress(
	progress,
	startTime,
	formatter,
	);
	if (indicator != null) eventListeners.add(indicator);

	if (printTiming) {
	eventListeners.add(TimingPrinter(startTime));
	}

	eventListeners.add(SkippedCompilationsPrinter());

	if (progress == Progress.status) {
	eventListeners.add(TimedProgressPrinter());
	}

	if (firstConf.reportFailures) {
	eventListeners.add(FailedTestsPrinter());
	}

	eventListeners.add(ResultCountPrinter(formatter));
	}

	if (firstConf.writeResults) {
	eventListeners.add(ResultWriter(firstConf.outputDirectory));
	}

	if (firstConf.copyCoreDumps) {
	eventListeners.add(UnexpectedCrashLogger());
	}

	// The only progress indicator when listing tests should be the
	// the summary printer.
	if (listTests) {
	eventListeners.add(SummaryPrinter(jsonOnly: reportInJson));
	} else {
	if (!firstConf.cleanExit) {
	eventListeners.add(ExitCodeSetter());
	}
	eventListeners.add(IgnoredTestMonitor());
	}

	// If any of the configurations need to access android devices we'll first
	// make a pool of all available adb devices.
	AdbDevicePool? adbDevicePool;
	var needsAdbDevicePool = configurations.any((conf) {
	return conf.system == System.android;
	});
	if (needsAdbDevicePool) {
	adbDevicePool = await AdbDevicePool.create();
	}

	// Start all the HTTP servers required before starting the process queue.
	if (serverFutures.isNotEmpty) {
	await Future.wait(serverFutures);
	}

	// [firstConf] is needed here, because the ProcessQueue uses some settings.
	ProcessQueue(
	firstConf,
	maxProcesses,
	maxBrowserProcesses,
	testSuites,
	eventListeners,
	allTestsFinished,
	verbose,
	adbDevicePool,
	);
	}