tools/testing/dart/configuration.dart

// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'compiler_configuration.dart';
import 'http_server.dart';
import 'path.dart';
import 'repository.dart';
import 'runtime_configuration.dart';

/// All of the contextual information to determine how a test suite should be
/// run.
///
/// Includes the compiler used to compile the code, the runtime the result is
/// executed on, etc.
class Configuration {
  Configuration(
      {this.architecture,
      this.compiler,
      this.mode,
      this.progress,
      this.runtime,
      this.system,
      this.selectors,
      this.appendLogs,
      this.batch,
      this.batchDart2JS,
      this.copyCoreDumps,
      this.hotReload,
      this.hotReloadRollback,
      this.isChecked,
      bool isStrong,
      this.isHostChecked,
      this.isCsp,
      this.isMinified,
      this.isVerbose,
      this.listTests,
      this.listStatusFiles,
      this.noPreviewDart2,
      this.printTiming,
      this.printReport,
      this.reportInJson,
      this.resetBrowser,
      this.skipCompilation,
      this.useAnalyzerCfe,
      this.useAnalyzerFastaParser,
      this.useBlobs,
      this.useSdk,
      this.useFastStartup,
      this.useEnableAsserts,
      this.useDart2JSWithKernel,
      this.useDart2JSOldFrontend,
      this.writeDebugLog,
      this.writeTestOutcomeLog,
      this.writeResultLog,
      this.drtPath,
      this.chromePath,
      this.safariPath,
      this.firefoxPath,
      this.dartPath,
      this.dartPrecompiledPath,
      this.flutterPath,
      this.taskCount,
      int timeout,
      this.shardCount,
      this.shard,
      this.stepName,
      this.testServerPort,
      this.testServerCrossOriginPort,
      this.testDriverErrorPort,
      this.localIP,
      this.dart2jsOptions,
      this.vmOptions,
      String packages,
      this.packageRoot,
      this.suiteDirectory,
      this.builderTag,
      this.outputDirectory,
      this.reproducingArguments,
      this.fastTestsOnly,
      this.printPassingStdout})
      : _packages = packages,
        _timeout = timeout,
        isStrong = isStrong ||
            // DDC always runs in strong mode.
            compiler == Compiler.dartdevc ||
            compiler == Compiler.dartdevk;

  final Architecture architecture;
  final Compiler compiler;
  final Mode mode;
  final Progress progress;
  final Runtime runtime;
  final System system;

  final Map<String, RegExp> selectors;

  // Boolean flags.

  final bool appendLogs;
  final bool batch;
  final bool batchDart2JS;
  final bool copyCoreDumps;
  final bool fastTestsOnly;
  final bool hotReload;
  final bool hotReloadRollback;
  final bool isChecked;
  final bool isStrong;
  final bool isHostChecked;
  final bool isCsp;
  final bool isMinified;
  final bool isVerbose;
  final bool listTests;
  final bool listStatusFiles;
  final bool noPreviewDart2;
  final bool printTiming;
  final bool printReport;
  final bool reportInJson;
  final bool resetBrowser;
  final bool skipCompilation;
  final bool useAnalyzerCfe;
  final bool useAnalyzerFastaParser;
  final bool useBlobs;
  final bool useSdk;
  final bool useFastStartup;
  final bool useEnableAsserts;
  final bool useDart2JSWithKernel;
  final bool useDart2JSOldFrontend;
  final bool writeDebugLog;
  final bool writeTestOutcomeLog;
  final bool writeResultLog;
  final bool printPassingStdout;

  // Various file paths.

  final String drtPath;
  final String chromePath;
  final String safariPath;
  final String firefoxPath;
  final String dartPath;
  final String dartPrecompiledPath;
  final String flutterPath;

  final int taskCount;
  final int shardCount;
  final int shard;
  final String stepName;

  final int testServerPort;
  final int testServerCrossOriginPort;
  final int testDriverErrorPort;
  final String localIP;

  /// Extra dart2js options passed to the testing script.
  final List<String> dart2jsOptions;

  /// Extra VM options passed to the testing script.
  final List<String> vmOptions;

  String _packages;
  String get packages {
    // If the .packages file path wasn't given, find it.
    if (packageRoot == null && _packages == null) {
      _packages = Repository.uri.resolve('.packages').toFilePath();
    }

    return _packages;
  }

  final String outputDirectory;
  final String packageRoot;
  final String suiteDirectory;
  final String builderTag;
  final List<String> reproducingArguments;

  TestingServers _servers;
  TestingServers get servers {
    if (_servers == null) {
      throw new StateError("Servers have not been started yet.");
    }
    return _servers;
  }

  /// Returns true if this configuration uses the new front end (fasta)
  /// as the first stage of compilation.
  bool get usesFasta {
    var fastaCompilers = const [
      Compiler.appJitk,
      Compiler.dartdevk,
      Compiler.dartk,
      Compiler.dartkp,
      Compiler.fasta,
    ];
    return fastaCompilers.contains(compiler) ||
        (compiler == Compiler.dart2js && !useDart2JSOldFrontend) ||
        (compiler == Compiler.dart2analyzer &&
            (builderTag == 'analyzer_use_fasta' || useAnalyzerCfe));
  }

  /// The base directory named for this configuration, like:
  ///
  ///     none_vm_release_x64
  String _configurationDirectory;
  String get configurationDirectory {
    // Lazy initialize and cache since it requires hitting the file system.
    if (_configurationDirectory == null) {
      _configurationDirectory = _calculateDirectory();
    }

    return _configurationDirectory;
  }

  /// The build directory path for this configuration, like:
  ///
  ///     build/none_vm_release_x64
  String get buildDirectory => system.outputDirectory + configurationDirectory;

  int _timeout;
  int get timeout {
    if (_timeout == null) {
      var isReload = hotReload || hotReloadRollback;

      var compilerMulitiplier = compilerConfiguration.timeoutMultiplier;
      var runtimeMultiplier = runtimeConfiguration.timeoutMultiplier(
          mode: mode,
          isChecked: isChecked,
          isReload: isReload,
          arch: architecture);

      _timeout = 60 * compilerMulitiplier * runtimeMultiplier;
    }

    return _timeout;
  }

  List<String> get standardOptions {
    if (compiler != Compiler.dart2js) {
      return const ["--ignore-unrecognized-flags"];
    }

    var args = ['--generate-code-with-compile-time-errors', '--test-mode'];
    if (isChecked) args.add('--enable-checked-mode');

    if (!runtime.isBrowser) {
      args.add("--allow-mock-compilation");
      args.add("--categories=all");
    }

    if (isMinified) args.add("--minify");
    if (isCsp) args.add("--csp");
    if (useFastStartup) args.add("--fast-startup");
    if (useEnableAsserts) args.add("--enable-asserts");
    if (useDart2JSWithKernel) args.add("--use-kernel");
    if (useDart2JSOldFrontend) args.add("--use-old-frontend");
    if (isStrong) args.add("--strong");
    return args;
  }

  String _windowsSdkPath;
  String get windowsSdkPath {
    if (!Platform.isWindows) {
      throw new StateError(
          "Should not use windowsSdkPath when not running on Windows.");
    }

    if (_windowsSdkPath == null) {
      // When running tests on Windows, use cdb from depot_tools to dump
      // stack traces of tests timing out.
      try {
        var path = new Path("build/win_toolchain.json").toNativePath();
        var text = new File(path).readAsStringSync();
        _windowsSdkPath = jsonDecode(text)['win_sdk'] as String;
      } on dynamic {
        // Ignore errors here. If win_sdk is not found, stack trace dumping
        // for timeouts won't work.
      }
    }

    return _windowsSdkPath;
  }

  /// Gets the local file path to the browser executable for this configuration.
  String get browserLocation {
    // If the user has explicitly configured a browser path, use it.
    String location;
    switch (runtime) {
      case Runtime.chrome:
        location = chromePath;
        break;
      case Runtime.drt:
        location = drtPath;
        break;
      case Runtime.firefox:
        location = firefoxPath;
        break;
      case Runtime.flutter:
        location = flutterPath;
        break;
      case Runtime.safari:
        location = safariPath;
        break;
    }

    if (location != null) return location;

    const locations = const {
      Runtime.firefox: const {
        System.windows: 'C:\\Program Files (x86)\\Mozilla Firefox\\firefox.exe',
        System.linux: 'firefox',
        System.macos: '/Applications/Firefox.app/Contents/MacOS/firefox'
      },
      Runtime.chrome: const {
        System.windows:
            'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
        System.macos:
            '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
        System.linux: 'google-chrome'
      },
      Runtime.safari: const {
        System.macos: '/Applications/Safari.app/Contents/MacOS/Safari'
      },
      Runtime.ie9: const {
        System.windows: 'C:\\Program Files\\Internet Explorer\\iexplore.exe'
      },
      Runtime.ie10: const {
        System.windows: 'C:\\Program Files\\Internet Explorer\\iexplore.exe'
      },
      Runtime.ie11: const {
        System.windows: 'C:\\Program Files\\Internet Explorer\\iexplore.exe'
      }
    };

    location = locations[runtime][System.find(Platform.operatingSystem)];

    if (location == null) {
      throw "${runtime.name} is not supported on ${Platform.operatingSystem}";
    }

    return location;
  }

  RuntimeConfiguration _runtimeConfiguration;
  RuntimeConfiguration get runtimeConfiguration =>
      _runtimeConfiguration ??= new RuntimeConfiguration(this);

  CompilerConfiguration _compilerConfiguration;
  CompilerConfiguration get compilerConfiguration =>
      _compilerConfiguration ??= new CompilerConfiguration(this);

  /// Determines if this configuration has a compatible compiler and runtime
  /// and other valid fields.
  ///
  /// Prints a warning if the configuration isn't valid. Returns whether or not
  /// it is.
  bool validate() {
    var isValid = true;
    var validRuntimes = compiler.supportedRuntimes;

    if (!validRuntimes.contains(runtime)) {
      print("Warning: combination of compiler '${compiler.name}' and "
          "runtime '${runtime.name}' is invalid. Skipping this combination.");
      isValid = false;
    }

    if (runtime.isIE &&
        Platform.operatingSystem != 'windows' &&
        !listStatusFiles &&
        !listTests) {
      print("Warning: cannot run Internet Explorer on non-Windows operating"
          " system.");
      isValid = false;
    }

    if (shard < 1 || shard > shardCount) {
      print("Error: shard index is $shard out of $shardCount shards");
      isValid = false;
    }

    if (runtime == Runtime.flutter && flutterPath == null) {
      print("-rflutter requires the flutter engine executable to "
          "be specified using --flutter");
      isValid = false;
    }

    if (runtime == Runtime.flutter && architecture != Architecture.x64) {
      isValid = false;
      print("-rflutter is applicable only for --arch=x64");
    }

    return isValid;
  }

  /// Starts global HTTP servers that serve the entire dart repo.
  ///
  /// The HTTP server is available on `window.location.port`, and a second
  /// server for cross-domain tests can be found by calling
  /// `getCrossOriginPortNumber()`.
  Future startServers() {
    _servers = new TestingServers(
        buildDirectory, isCsp, runtime, null, packageRoot, packages);
    var future = servers.startServers(localIP,
        port: testServerPort, crossOriginPort: testServerCrossOriginPort);

    if (isVerbose) {
      future = future.then((_) {
        print('Started HttpServers: ${servers.commandLine}');
      });
    }

    return future;
  }

  void stopServers() {
    if (_servers != null) _servers.stopServers();
  }

  /// Returns the correct configuration directory (the last component of the
  /// output directory path) for regular dart checkouts.
  ///
  /// We allow our code to have been cross compiled, i.e., that there is an X
  /// in front of the arch. We don't allow both a cross compiled and a normal
  /// version to be present (except if you specifically pass in the
  /// build_directory).
  String _calculateDirectory() {
    // Capitalize the mode name.
    var modeName =
        mode.name.substring(0, 1).toUpperCase() + mode.name.substring(1);

    var os = '';
    if (system == System.android) os = "Android";

    var arch = architecture.name.toUpperCase();
    var normal = '$modeName$os$arch';
    var cross = '$modeName${os}X$arch';
    var outDir = system.outputDirectory;
    var normalDir = new Directory(new Path('$outDir$normal').toNativePath());
    var crossDir = new Directory(new Path('$outDir$cross').toNativePath());

    if (normalDir.existsSync() && crossDir.existsSync()) {
      throw "You can't have both $normalDir and $crossDir. We don't know which"
          " binary to use.";
    }

    if (crossDir.existsSync()) return cross;

    return normal;
  }

  Map _summaryMap;

  /// [toSummaryMap] returns a map of configurations important to the running
  /// of a test. Flags and properties used for output are not included.
  /// The summary map can be used to serialize to json for test-output logging.
  Map toSummaryMap() {
    if (_summaryMap == null) {
      _summaryMap = {
        'mode': mode.name,
        'arch': architecture.name,
        'compiler': compiler.name,
        'runtime': runtime.name,
        'checked': isChecked,
        'strong': isStrong,
        'host_checked': isHostChecked,
        'minified': isMinified,
        'csp': isCsp,
        'system': system.name,
        'vm_options': vmOptions,
        'fasta': usesFasta,
        'use_sdk': useSdk,
        'builder_tag': builderTag,
        'fast_startup': useFastStartup,
        'timeout': timeout,
        'no_preview_dart_2': noPreviewDart2,
        'use_cfe': useAnalyzerCfe,
        'analyzer_use_fasta_parser': useAnalyzerFastaParser,
        'dart2js_with_kernel': useDart2JSWithKernel,
        'dart2js_old_frontend': useDart2JSOldFrontend,
        'enable_asserts': useEnableAsserts,
        'hot_reload': hotReload,
        'hot_reload_rollback': hotReloadRollback,
        'batch': batch,
        'batch_dart2js': batchDart2JS,
        'reset_browser_configuration': resetBrowser,
        'selectors': selectors.keys.toList()
      };
    }
    return _summaryMap;
  }
}

class Architecture {
  static const ia32 = const Architecture._('ia32');
  static const x64 = const Architecture._('x64');
  static const arm = const Architecture._('arm');
  static const armv6 = const Architecture._('armv6');
  static const armv5te = const Architecture._('armv5te');
  static const arm64 = const Architecture._('arm64');
  static const simarm = const Architecture._('simarm');
  static const simarmv6 = const Architecture._('simarmv6');
  static const simarmv5te = const Architecture._('simarmv5te');
  static const simarm64 = const Architecture._('simarm64');
  static const simdbc = const Architecture._('simdbc');
  static const simdbc64 = const Architecture._('simdbc64');

  static final List<String> names = _all.keys.toList();

  static final _all = new Map<String, Architecture>.fromIterable([
    ia32,
    x64,
    arm,
    armv6,
    armv5te,
    arm64,
    simarm,
    simarmv6,
    simarmv5te,
    simarm64,
    simdbc,
    simdbc64
  ], key: (architecture) => (architecture as Architecture).name);

  static Architecture find(String name) {
    var architecture = _all[name];
    if (architecture != null) return architecture;

    throw new ArgumentError('Unknown architecture "$name".');
  }

  final String name;

  const Architecture._(this.name);

  String toString() => "Architecture($name)";
}

class Compiler {
  static const none = const Compiler._('none');
  static const precompiler = const Compiler._('precompiler');
  static const dart2js = const Compiler._('dart2js');
  static const dart2analyzer = const Compiler._('dart2analyzer');
  static const dartdevc = const Compiler._('dartdevc');
  static const dartdevk = const Compiler._('dartdevk');
  static const appJit = const Compiler._('app_jit');
  static const appJitk = const Compiler._('app_jitk');
  static const dartk = const Compiler._('dartk');
  static const dartkp = const Compiler._('dartkp');
  static const specParser = const Compiler._('spec_parser');
  static const fasta = const Compiler._('fasta');

  static final List<String> names = _all.keys.toList();

  static final _all = new Map<String, Compiler>.fromIterable([
    none,
    precompiler,
    dart2js,
    dart2analyzer,
    dartdevc,
    dartdevk,
    appJit,
    appJitk,
    dartk,
    dartkp,
    specParser,
    fasta,
  ], key: (compiler) => (compiler as Compiler).name);

  static Compiler find(String name) {
    var compiler = _all[name];
    if (compiler != null) return compiler;

    throw new ArgumentError('Unknown compiler "$name".');
  }

  final String name;

  const Compiler._(this.name);

  /// Gets the runtimes this compiler can target.
  List<Runtime> get supportedRuntimes {
    switch (this) {
      case Compiler.dart2js:
        // Note: by adding 'none' as a configuration, if the user
        // runs test.py -c dart2js -r drt,none the dart2js_none and
        // dart2js_drt will be duplicating work. If later we don't need 'none'
        // with dart2js, we should remove it from here.
        return const [
          Runtime.d8,
          Runtime.jsshell,
          Runtime.drt,
          Runtime.none,
          Runtime.firefox,
          Runtime.chrome,
          Runtime.safari,
          Runtime.ie9,
          Runtime.ie10,
          Runtime.ie11,
          Runtime.opera,
          Runtime.chromeOnAndroid,
        ];

      case Compiler.dartdevc:
      case Compiler.dartdevk:
        // TODO(rnystrom): Expand to support other JS execution environments
        // (other browsers, d8) when tested and working.
        return const [
          Runtime.none,
          Runtime.drt,
          Runtime.chrome,
        ];

      case Compiler.dart2analyzer:
        return const [Runtime.none];
      case Compiler.appJit:
      case Compiler.appJitk:
      case Compiler.dartk:
        return const [Runtime.vm, Runtime.selfCheck];
      case Compiler.precompiler:
      case Compiler.dartkp:
        return const [Runtime.dartPrecompiled];
      case Compiler.specParser:
        return const [Runtime.none];
      case Compiler.fasta:
        return const [Runtime.none];
      case Compiler.none:
        return const [
          Runtime.vm,
          Runtime.flutter,
          Runtime.drt,
          Runtime.contentShellOnAndroid
        ];
    }

    throw "unreachable";
  }

  /// The preferred runtime to use with this compiler if no other runtime is
  /// specified.
  Runtime get defaultRuntime {
    switch (this) {
      case Compiler.dart2js:
        return Runtime.d8;
      case Compiler.dartdevc:
      case Compiler.dartdevk:
        return Runtime.chrome;
      case Compiler.dart2analyzer:
        return Runtime.none;
      case Compiler.appJit:
      case Compiler.appJitk:
      case Compiler.dartk:
        return Runtime.vm;
      case Compiler.precompiler:
      case Compiler.dartkp:
        return Runtime.dartPrecompiled;
      case Compiler.specParser:
      case Compiler.fasta:
        return Runtime.none;
      case Compiler.none:
        return Runtime.vm;
    }

    throw "unreachable";
  }

  Mode get defaultMode {
    switch (this) {
      case Compiler.dart2analyzer:
      case Compiler.dart2js:
      case Compiler.dartdevc:
      case Compiler.dartdevk:
      case Compiler.fasta:
        return Mode.release;

      default:
        return Mode.debug;
    }
  }

  String toString() => "Compiler($name)";
}

class Mode {
  static const debug = const Mode._('debug');
  static const product = const Mode._('product');
  static const release = const Mode._('release');

  static final List<String> names = _all.keys.toList();

  static final _all = new Map<String, Mode>.fromIterable(
      [debug, product, release],
      key: (mode) => (mode as Mode).name);

  static Mode find(String name) {
    var mode = _all[name];
    if (mode != null) return mode;

    throw new ArgumentError('Unknown mode "$name".');
  }

  final String name;

  const Mode._(this.name);

  bool get isDebug => this == debug;

  String toString() => "Mode($name)";
}

class Progress {
  static const compact = const Progress._('compact');
  static const color = const Progress._('color');
  static const line = const Progress._('line');
  static const verbose = const Progress._('verbose');
  static const silent = const Progress._('silent');
  static const status = const Progress._('status');
  static const buildbot = const Progress._('buildbot');
  static const diff = const Progress._('diff');

  static final List<String> names = _all.keys.toList();

  static final _all = new Map<String, Progress>.fromIterable(
      [compact, color, line, verbose, silent, status, buildbot, diff],
      key: (progress) => (progress as Progress).name);

  static Progress find(String name) {
    var progress = _all[name];
    if (progress != null) return progress;

    throw new ArgumentError('Unknown progress type "$name".');
  }

  final String name;

  const Progress._(this.name);

  String toString() => "Progress($name)";
}

class Runtime {
  static const vm = const Runtime._('vm');
  static const flutter = const Runtime._('flutter');
  static const dartPrecompiled = const Runtime._('dart_precompiled');
  static const d8 = const Runtime._('d8');
  static const jsshell = const Runtime._('jsshell');
  static const drt = const Runtime._('drt');
  static const firefox = const Runtime._('firefox');
  static const chrome = const Runtime._('chrome');
  static const safari = const Runtime._('safari');
  static const ie9 = const Runtime._('ie9');
  static const ie10 = const Runtime._('ie10');
  static const ie11 = const Runtime._('ie11');
  static const opera = const Runtime._('opera');
  static const chromeOnAndroid = const Runtime._('chromeOnAndroid');
  static const contentShellOnAndroid = const Runtime._('ContentShellOnAndroid');
  static const selfCheck = const Runtime._('self_check');
  static const none = const Runtime._('none');

  static final List<String> names = _all.keys.toList()..add("ff");

  static final _all = new Map<String, Runtime>.fromIterable([
    vm,
    flutter,
    dartPrecompiled,
    d8,
    jsshell,
    drt,
    firefox,
    chrome,
    safari,
    ie9,
    ie10,
    ie11,
    opera,
    chromeOnAndroid,
    contentShellOnAndroid,
    selfCheck,
    none
  ], key: (runtime) => (runtime as Runtime).name);

  static Runtime find(String name) {
    // Allow "ff" as a synonym for Firefox.
    if (name == "ff") return firefox;

    var runtime = _all[name];
    if (runtime != null) return runtime;

    throw new ArgumentError('Unknown runtime "$name".');
  }

  final String name;

  const Runtime._(this.name);

  bool get isBrowser => const [
        drt,
        ie9,
        ie10,
        ie11,
        safari,
        opera,
        chrome,
        firefox,
        chromeOnAndroid,
        contentShellOnAndroid
      ].contains(this);

  bool get isIE => name.startsWith("ie");

  bool get isSafari => name.startsWith("safari");

  /// Whether this runtime is a command-line JavaScript environment.
  bool get isJSCommandLine => const [d8, jsshell].contains(this);

  /// If the runtime doesn't support `Window.open`, we use iframes instead.
  bool get requiresIFrame => !const [ie11, ie10].contains(this);

  /// The preferred compiler to use with this runtime if no other compiler is
  /// specified.
  Compiler get defaultCompiler {
    switch (this) {
      case vm:
      case flutter:
      case drt:
        return Compiler.none;

      case dartPrecompiled:
        return Compiler.precompiler;

      case d8:
      case jsshell:
      case firefox:
      case chrome:
      case safari:
      case ie9:
      case ie10:
      case ie11:
      case opera:
      case chromeOnAndroid:
      case contentShellOnAndroid:
        return Compiler.dart2js;

      case selfCheck:
        return Compiler.dartk;

      case none:
        // If we aren't running it, we probably just want to analyze it.
        return Compiler.dart2analyzer;
    }

    throw "unreachable";
  }

  String toString() => "Runtime($name)";
}

class System {
  static const android = const System._('android');
  static const fuchsia = const System._('fuchsia');
  static const linux = const System._('linux');
  static const macos = const System._('macos');
  static const windows = const System._('windows');

  static final List<String> names = _all.keys.toList();

  static final _all = new Map<String, System>.fromIterable(
      [android, fuchsia, linux, macos, windows],
      key: (system) => (system as System).name);

  static System find(String name) {
    var system = _all[name];
    if (system != null) return system;

    throw new ArgumentError('Unknown operating system "$name".');
  }

  final String name;

  const System._(this.name);

  /// The root directory name for build outputs on this system.
  String get outputDirectory {
    switch (this) {
      case android:
      case fuchsia:
      case linux:
      case windows:
        return 'out/';

      case macos:
        return 'xcodebuild/';
    }

    throw "unreachable";
  }

  String toString() => "System($name)";
}