// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/**
 * Classes and methods for executing tests.
 *
 * This module includes:
 * - Managing parallel execution of tests, including timeout checks.
 * - Evaluating the output of each test as pass/fail/crash/timeout.
 */
library test_runner;

import "dart:async";
import "dart:collection" show Queue;
// We need to use the 'io' prefix here, otherwise io.exitCode will shadow
// CommandOutput.exitCode in subclasses of CommandOutput.
import "dart:io" as io;
import "dart:isolate";
import "dart:uri";
import "http_server.dart" as http_server;
import "status_file_parser.dart";
import "test_progress.dart";
import "test_suite.dart";
import "utils.dart";

const int NO_TIMEOUT = 0;
const int SLOW_TIMEOUT_MULTIPLIER = 4;

const int CRASHING_BROWSER_EXITCODE = -10;

typedef void TestCaseEvent(TestCase testCase);
typedef void ExitCodeEvent(int exitCode);
typedef void EnqueueMoreWork(ProcessQueue queue);


/**
 * [areByteArraysEqual] compares a range of bytes from [buffer1] with a
 * range of bytes from [buffer2].
 *
 * Returns [true] if the [count] bytes in [buffer1] (starting at
 * [offset1]) match the [count] bytes in [buffer2] (starting at
 * [offset2]).
 * Otherwise [false] is returned.
 */
bool areByteArraysEqual(List<int> buffer1, int offset1,
                        List<int> buffer2, int offset2,
                        int count) {
  if ((offset1 + count) > buffer1.length ||
      (offset2 + count) > buffer2.length) {
    return false;
  }

  for (var i = 0; i < count; i++) {
    if (buffer1[offset1 + i] != buffer2[offset2 + i]) {
      return false;
    }
  }
  return true;
}

/**
 * [findBytes] searches for [pattern] in [data] beginning at [startPos].
 *
 * Returns [true] if [pattern] was found in [data].
 * Otherwise [false] is returned.
 */
int findBytes(List<int> data, List<int> pattern, [int startPos=0]) {
  // TODO(kustermann): Use one of the fast string-matching algorithms!
  for (int i=startPos; i < (data.length-pattern.length); i++) {
    bool found = true;
    for (int j=0; j<pattern.length; j++) {
      if (data[i+j] != pattern[j]) {
        found = false;
      }
    }
    if (found) {
      return i;
    }
  }
  return -1;
}


/** A command executed as a step in a test case. */
class Command {
  static int nextHashCode = 0;
  final int hashCode = nextHashCode++;
  operator ==(other) => super == (other);

  /** Path to the executable of this command. */
  String executable;

  /** Command line arguments to the executable. */
  List<String> arguments;

  /** Environment for the command */
  Map<String,String> environment;

  /** The actual command line that will be executed. */
  String commandLine;

  Command(this.executable, this.arguments, [this.environment = null]) {
    if (io.Platform.operatingSystem == 'windows') {
      // Windows can't handle the first command if it is a .bat file or the like
      // with the slashes going the other direction.
      // TODO(efortuna): Remove this when fixed (Issue 1306).
      executable = executable.replaceAll('/', '\\');
    }
    commandLine = "$executable ${arguments.join(' ')}";
  }

  String toString() => commandLine;

  Future<bool> get outputIsUpToDate => new Future.immediate(false);
  io.Path get expectedOutputFile => null;
  bool get isPixelTest => false;
}

class CompilationCommand extends Command {
  String _outputFile;
  bool _neverSkipCompilation;
  List<Uri> _bootstrapDependencies;

  CompilationCommand(this._outputFile,
                     this._neverSkipCompilation,
                     this._bootstrapDependencies,
                     String executable,
                     List<String> arguments)
      : super(executable, arguments);

  Future<bool> get outputIsUpToDate {
    if (_neverSkipCompilation) return new Future.immediate(false);

    Future<List<Uri>> readDepsFile(String path) {
      var file = new io.File(new io.Path(path).toNativePath());
      if (!file.existsSync()) {
        return new Future.immediate(null);
      }
      return file.readAsLines().then((List<String> lines) {
        var dependencies = new List<Uri>();
        for (var line in lines) {
          line = line.trim();
          if (line.length > 0) {
            dependencies.add(new Uri(line));
          }
        }
        return dependencies;
      });
    }

    return readDepsFile("$_outputFile.deps").then((dependencies) {
      if (dependencies != null) {
        dependencies.addAll(_bootstrapDependencies);
        var jsOutputLastModified = TestUtils.lastModifiedCache.getLastModified(
            new Uri.fromComponents(scheme: 'file', path: _outputFile));
        if (jsOutputLastModified != null) {
          for (var dependency in dependencies) {
            var dependencyLastModified =
                TestUtils.lastModifiedCache.getLastModified(dependency);
            if (dependencyLastModified == null ||
                dependencyLastModified > jsOutputLastModified) {
              return false;
            }
          }
          return true;
        }
      }
      return false;
    });
  }
}

class DumpRenderTreeCommand extends Command {
  /**
   * If [expectedOutputPath] is set, the output of DumpRenderTree is compared
   * with the content of [expectedOutputPath].
   * This is used for example for pixel tests, where [expectedOutputPath] points
   * to a *png file.
   */
  io.Path expectedOutputPath;

  DumpRenderTreeCommand(String executable,
                        String htmlFile,
                        List<String> options,
                        List<String> dartFlags,
                        io.Path this.expectedOutputPath)
      : super(executable,
              _getArguments(options, htmlFile),
              _getEnvironment(dartFlags));

  static Map _getEnvironment(List<String> dartFlags) {
    var needDartFlags = dartFlags != null && dartFlags.length > 0;

    var env = null;
    if (needDartFlags) {
      env = new Map.from(io.Platform.environment);
      if (needDartFlags) {
        env['DART_FLAGS'] = dartFlags.join(" ");
      }
    }

    return env;
  }

  static List<String> _getArguments(List<String> options, String htmlFile) {
    var arguments = new List.from(options);
    arguments.add(htmlFile);
    return arguments;
  }

  io.Path get expectedOutputFile => expectedOutputPath;
  bool get isPixelTest => (expectedOutputFile != null &&
                           expectedOutputFile.filename.endsWith(".png"));
}


/**
 * TestCase contains all the information needed to run a test and evaluate
 * its output.  Running a test involves starting a separate process, with
 * the executable and arguments given by the TestCase, and recording its
 * stdout and stderr output streams, and its exit code.  TestCase only
 * contains static information about the test; actually running the test is
 * performed by [ProcessQueue] using a [RunningProcess] object.
 *
 * The output information is stored in a [CommandOutput] instance contained
 * in TestCase.commandOutputs. The last CommandOutput instance is responsible
 * for evaluating if the test has passed, failed, crashed, or timed out, and the
 * TestCase has information about what the expected result of the test should
 * be.
 *
 * The TestCase has a callback function, [completedHandler], that is run when
 * the test is completed.
 */
class TestCase {
  /**
   * A list of commands to execute. Most test cases have a single command.
   * Dart2js tests have two commands, one to compile the source and another
   * to execute it. Some isolate tests might even have three, if they require
   * compiling multiple sources that are run in isolation.
   */
  List<Command> commands;
  Map<Command, CommandOutput> commandOutputs = new Map<Command,CommandOutput>();

  Map configuration;
  String displayName;
  bool isNegative;
  Set<String> expectedOutcomes;
  TestCaseEvent completedHandler;
  TestInformation info;

  TestCase(this.displayName,
           this.commands,
           this.configuration,
           this.completedHandler,
           this.expectedOutcomes,
           {this.isNegative: false,
            this.info: null}) {
    if (!isNegative) {
      this.isNegative = displayName.contains("negative_test");
    }

    // Special command handling. If a special command is specified
    // we have to completely rewrite the command that we are using.
    // We generate a new command-line that is the special command where we
    // replace '@' with the original command executable, and generate
    // a command formed like the following
    // Let PREFIX be what is before the @.
    // Let SUFFIX be what is after the @.
    // Let EXECUTABLE be the existing executable of the command.
    // Let ARGUMENTS be the existing arguments to the existing executable.
    // The new command will be:
    // PREFIX EXECUTABLE SUFFIX ARGUMENTS
    var specialCommand = configuration['special-command'];
    if (!specialCommand.isEmpty) {
      Expect.isTrue(specialCommand.contains('@'),
                    "special-command must contain a '@' char");
      var specialCommandSplit = specialCommand.split('@');
      var prefix = specialCommandSplit[0].trim();
      var suffix = specialCommandSplit[1].trim();
      List<Command> newCommands = [];
      for (Command c in commands) {
        // If we don't have a new prefix we will use the existing executable.
        var newExecutablePath = c.executable;;
        var newArguments = [];

        if (prefix.length > 0) {
          var prefixSplit = prefix.split(' ');
          newExecutablePath = prefixSplit[0];
          for (int i = 1; i < prefixSplit.length; i++) {
            var current = prefixSplit[i];
            if (!current.isEmpty) newArguments.add(current);
          }
          newArguments.add(c.executable);
        }

        // Add any suffixes to the arguments of the original executable.
        var suffixSplit = suffix.split(' ');
        suffixSplit.forEach((e) {
          if (!e.isEmpty) newArguments.add(e);
        });

        newArguments.addAll(c.arguments);
        final newCommand = new Command(newExecutablePath, newArguments);
        newCommands.add(newCommand);
        // If there are extra spaces inside the prefix or suffix, this fails.
        String expected =
            '$prefix ${c.executable} $suffix ${c.arguments.join(' ')}';
        Expect.stringEquals(expected.trim(), newCommand.commandLine);
      }
      commands = newCommands;
    }
  }

  CommandOutput get lastCommandOutput {
    if (commandOutputs.length == 0) {
      throw new Exception("CommandOutputs is empty, maybe no command was run? ("
                          "displayName: '$displayName', "
                          "configurationString: '$configurationString')");
    }
    return commandOutputs[commands[commandOutputs.length - 1]];
  }

  int get timeout {
    if (expectedOutcomes.contains(SLOW)) {
      return configuration['timeout'] * SLOW_TIMEOUT_MULTIPLIER;
    } else {
      return configuration['timeout'];
    }
  }

  String get configurationString {
    final compiler = configuration['compiler'];
    final runtime = configuration['runtime'];
    final mode = configuration['mode'];
    final arch = configuration['arch'];
    final checked = configuration['checked'] ? '-checked' : '';
    return "$compiler-$runtime$checked ${mode}_$arch";
  }

  List<String> get batchRunnerArguments => ['-batch'];
  List<String> get batchTestArguments => commands.last.arguments;

  bool get usesWebDriver => TestUtils.usesWebDriver(configuration['runtime']);

  void completed() { completedHandler(this); }

  bool get isFlaky {
      if (expectedOutcomes.contains(SKIP)) {
        return false;
      }

      var flags = new Set.from(expectedOutcomes);
      flags..remove(TIMEOUT)
           ..remove(SLOW);
      return flags.contains(PASS) && flags.length > 1;
  }
}


/**
 * BrowserTestCase has an extra compilation command that is run in a separate
 * process, before the regular test is run as in the base class [TestCase].
 * If the compilation command fails, then the rest of the test is not run.
 */
class BrowserTestCase extends TestCase {
  /**
   * Indicates the number of potential retries remaining, to compensate for
   * flaky browser tests.
   */
  int numRetries;

  /**
   * True if this test is dependent on another test completing before it can
   * star (for example, we might need to depend on some other test completing
   * first).
   */
  bool waitingForOtherTest;

  /**
   * The set of test cases that wish to be notified when this test has
   * completed.
   */
  List<BrowserTestCase> observers;

  BrowserTestCase(displayName, commands, configuration, completedHandler,
      expectedOutcomes, info, isNegative, [this.waitingForOtherTest = false])
    : super(displayName, commands, configuration, completedHandler,
        expectedOutcomes, isNegative: isNegative, info: info) {
    numRetries = 2; // Allow two retries to compensate for flaky browser tests.
    observers = [];
  }

  List<String> get _lastArguments => commands.last.arguments;

  List<String> get batchRunnerArguments => [_lastArguments[0], '--batch'];

  List<String> get batchTestArguments =>
      _lastArguments.getRange(1, _lastArguments.length - 1);

  /** Add a test case to listen for when this current test has completed. */
  void addObserver(BrowserTestCase testCase) {
    observers.add(testCase);
  }

  /**
   * Notify all of the test cases that are dependent on this one that they can
   * proceed.
   */
  void notifyObservers() {
    for (BrowserTestCase testCase in observers) {
      testCase.waitingForOtherTest = false;
    }
  }
}


/**
 * CommandOutput records the output of a completed command: the process's exit
 * code, the standard output and standard error, whether the process timed out,
 * and the time the process took to run.  It also contains a pointer to the
 * [TestCase] this is the output of.
 */
abstract class CommandOutput {
  factory CommandOutput.fromCase(TestCase testCase,
                                 Command command,
                                 int exitCode,
                                 bool incomplete,
                                 bool timedOut,
                                 List<int> stdout,
                                 List<int> stderr,
                                 Duration time,
                                 bool compilationSkipped) {
    return new CommandOutputImpl.fromCase(testCase,
                                          command,
                                          exitCode,
                                          incomplete,
                                          timedOut,
                                          stdout,
                                          stderr,
                                          time,
                                          compilationSkipped);
  }

  Command get command;

  bool get incomplete;

  String get result;

  bool get unexpectedOutput;

  bool get hasCrashed;

  bool get hasTimedOut;

  bool get didFail;

  bool requestRetry;

  Duration get time;

  int get exitCode;

  List<int> get stdout;

  List<int> get stderr;

  List<String> get diagnostics;

  bool get compilationSkipped;
}

class CommandOutputImpl implements CommandOutput {
  Command command;
  TestCase testCase;
  int exitCode;

  /// Records if all commands were run, true if they weren't.
  final bool incomplete;

  bool timedOut;
  bool failed = false;
  List<int> stdout;
  List<int> stderr;
  Duration time;
  List<String> diagnostics;
  bool compilationSkipped;

  /**
   * A flag to indicate we have already printed a warning about ignoring the VM
   * crash, to limit the amount of output produced per test.
   */
  bool alreadyPrintedWarning = false;

  /**
   * Set to true if we encounter a condition in the output that indicates we
   * need to rerun this test.
   */
  bool requestRetry = false;

  // Don't call this constructor, call CommandOutput.fromCase() to
  // get a new TestOutput instance.
  CommandOutputImpl(TestCase this.testCase,
                    Command this.command,
                    int this.exitCode,
                    bool this.incomplete,
                    bool this.timedOut,
                    List<int> this.stdout,
                    List<int> this.stderr,
                    Duration this.time,
                    bool this.compilationSkipped) {
    testCase.commandOutputs[command] = this;
    diagnostics = [];
  }
  factory CommandOutputImpl.fromCase(TestCase testCase,
                                     Command command,
                                     int exitCode,
                                     bool incomplete,
                                     bool timedOut,
                                     List<int> stdout,
                                     List<int> stderr,
                                     Duration time,
                                     bool compilationSkipped) {
    if (testCase is BrowserTestCase) {
      return new BrowserCommandOutputImpl(testCase,
                                          command,
                                          exitCode,
                                          incomplete,
                                          timedOut,
                                          stdout,
                                          stderr,
                                          time,
                                          compilationSkipped);
    } else if (testCase.configuration['compiler'] == 'dartc') {
      return new AnalysisCommandOutputImpl(testCase,
                                           command,
                                           exitCode,
                                           timedOut,
                                           stdout,
                                           stderr,
                                           time,
                                           compilationSkipped);
    }
    return new CommandOutputImpl(testCase,
                                 command,
                                 exitCode,
                                 incomplete,
                                 timedOut,
                                 stdout,
                                 stderr,
                                 time,
                                 compilationSkipped);
  }

  String get result =>
      hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS));

  bool get unexpectedOutput => !testCase.expectedOutcomes.contains(result);

  bool get hasCrashed {
    // The Java dartc runner and dart2js exits with code 253 in case
    // of unhandled exceptions.
    if (exitCode == 253) return true;
    if (io.Platform.operatingSystem == 'windows') {
      // The VM uses std::abort to terminate on asserts.
      // std::abort terminates with exit code 3 on Windows.
      if (exitCode == 3) {
        return !timedOut;
      }
      // TODO(ricow): Remove this dirty hack ones we have a selenium
      // replacement.
      if (exitCode == CRASHING_BROWSER_EXITCODE) {
        return !timedOut;
      }
      // If a program receives an uncaught system exception, the program
      // terminates with the exception code as exit code.
      // The 0x3FFFFF00 mask here tries to determine if an exception indicates
      // a crash of the program.
      // System exception codes can be found in 'winnt.h', for example
      // "#define STATUS_ACCESS_VIOLATION  ((DWORD) 0xC0000005)"
      return (!timedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0));
    }
    return !timedOut && ((exitCode < 0));
  }

  bool get hasTimedOut => timedOut;

  bool get didFail {
    return (exitCode != 0 && !hasCrashed);
  }

  // Reverse result of a negative test.
  bool get hasFailed {
    // Always fail if a runtime-error is expected and compilation failed.
    if (testCase.info != null && testCase.info.hasRuntimeError && incomplete) {
      return true;
    }
    return testCase.isNegative ? !didFail : didFail;
  }

}

class BrowserCommandOutputImpl extends CommandOutputImpl {
  BrowserCommandOutputImpl(
      testCase,
      command,
      exitCode,
      incomplete,
      timedOut,
      stdout,
      stderr,
      time,
      compilationSkipped) :
    super(testCase,
          command,
          exitCode,
          incomplete,
          timedOut,
          stdout,
          stderr,
          time,
          compilationSkipped);

  bool get didFail {
    if (_failedBecauseOfMissingXDisplay) {
      return true;
    }

    if (command.expectedOutputFile != null) {
      // We are either doing a pixel test or a layout test with DumpRenderTree
      return _failedBecauseOfUnexpectedDRTOutput;
    }
    return _browserTestFailure;
  }

  bool get _failedBecauseOfMissingXDisplay {
    // Browser case:
    // If the browser test failed, it may have been because DumpRenderTree
    // and the virtual framebuffer X server didn't hook up, or DRT crashed with
    // a core dump. Sometimes DRT crashes after it has set the stdout to PASS,
    // so we have to do this check first.
    var stderrLines = decodeUtf8(super.stderr).split("\n");
    for (String line in stderrLines) {
      // TODO(kustermann,ricow): Issue: 7564
      // This seems to happen quite frequently, we need to figure out why.
      if (line.contains('Gtk-WARNING **: cannot open display') ||
          line.contains('Failed to run command. return code=1')) {
        // If we get the X server error, or DRT crashes with a core dump, retry
        // the test.
        if ((testCase as BrowserTestCase).numRetries > 0) {
          requestRetry = true;
        }
        print("Warning: Test failure because of missing XDisplay");
        return true;
      }
    }
    return false;
  }

  bool get _failedBecauseOfUnexpectedDRTOutput {
    /*
     * The output of DumpRenderTree is different for pixel tests than for
     * layout tests.
     *
     * On a pixel test, the DRT output has the following format
     *     ......
     *     ......
     *     Content-Length: ...\n
     *     <*png data>
     *     #EOF\n
     * So we need to get the byte-range of the png data first, before
     * comparing it with the content of the expected output file.
     *
     * On a layout tests, the DRT output is directly compared with the
     * content of the expected output.
     */
    var stdout = testCase.commandOutputs[command].stdout;
    var file = new io.File.fromPath(command.expectedOutputFile);
    if (file.existsSync()) {
      var bytesContentLength = "Content-Length:".charCodes;
      var bytesNewLine = "\n".charCodes;
      var bytesEOF = "#EOF\n".charCodes;

      var expectedContent = file.readAsBytesSync();
      if (command.isPixelTest) {
        var startOfContentLength = findBytes(stdout, bytesContentLength);
        if (startOfContentLength >= 0) {
          var newLineAfterContentLength = findBytes(stdout,
                                                    bytesNewLine,
                                                    startOfContentLength);
          if (newLineAfterContentLength > 0) {
            var startPosition = newLineAfterContentLength +
                bytesNewLine.length;
            var endPosition = stdout.length - bytesEOF.length;

            return !areByteArraysEqual(expectedContent,
                                       0,
                                       stdout,
                                       startPosition,
                                       endPosition - startPosition);
          }
        }
        return true;
      } else {
        return !areByteArraysEqual(expectedContent, 0,
                                   stdout, 0,
                                   stdout.length);
      }
    }
    return true;
  }

  bool get _browserTestFailure {
    // Browser tests fail unless stdout contains
    // 'Content-Type: text/plain' followed by 'PASS'.
    bool has_content_type = false;
    var stdoutLines = decodeUtf8(super.stdout).split("\n");
    for (String line in stdoutLines) {
      switch (line) {
        case 'Content-Type: text/plain':
          has_content_type = true;
          break;
        case 'PASS':
          if (has_content_type) {
            if (exitCode != 0) {
              print("Warning: All tests passed, but exitCode != 0 "
                    "(${testCase.displayName})");
            }
            if (testCase.configuration['runtime'] == 'drt') {
              // TODO(kustermann/ricow): Issue: 7563
              // We should eventually get rid of this hack.
              return false;
            } else {
              return (exitCode != 0 && !hasCrashed);
            }
          }
          break;
      }
    }
    return true;
  }
}

// The static analyzer does not actually execute code, so
// the criteria for success now depend on the text sent
// to stderr.
class AnalysisCommandOutputImpl extends CommandOutputImpl {
  // An error line has 8 fields that look like:
  // ERROR|COMPILER|MISSING_SOURCE|file:/tmp/t.dart|15|1|24|Missing source.
  final int ERROR_LEVEL = 0;
  final int ERROR_TYPE = 1;
  final int FORMATTED_ERROR = 7;

  bool alreadyComputed = false;
  bool failResult;

  AnalysisCommandOutputImpl(testCase,
                            command,
                            exitCode,
                            timedOut,
                            stdout,
                            stderr,
                            time,
                            compilationSkipped) :
    super(testCase,
          command,
          exitCode,
          false,
          timedOut,
          stdout,
          stderr,
          time,
          compilationSkipped);

  bool get didFail {
    if (!alreadyComputed) {
      failResult = _didFail();
      alreadyComputed = true;
    }
    return failResult;
  }

  bool _didFail() {
    if (hasCrashed) return false;

    List<String> errors = [];
    List<String> staticWarnings = [];

    // Read the returned list of errors and stuff them away.
    var stderrLines = decodeUtf8(super.stderr).split("\n");
    for (String line in stderrLines) {
      if (line.length == 0) continue;
      List<String> fields = splitMachineError(line);
      if (fields[ERROR_LEVEL] == 'ERROR') {
        errors.add(fields[FORMATTED_ERROR]);
      } else if (fields[ERROR_LEVEL] == 'WARNING') {
        // We only care about testing Static type warnings
        // ignore all others
        if (fields[ERROR_TYPE] == 'STATIC_TYPE') {
          staticWarnings.add(fields[FORMATTED_ERROR]);
        }
      }
      // OK to Skip error output that doesn't match the machine format
    }
    if (testCase.info != null
        && testCase.info.optionsFromFile['isMultitest']) {
      return _didMultitestFail(errors, staticWarnings);
    }
    return _didStandardTestFail(errors, staticWarnings);
  }

  bool _didMultitestFail(List errors, List staticWarnings) {
    Set<String> outcome = testCase.info.multitestOutcome;
    Expect.isNotNull(outcome);
    if (outcome.contains('compile-time error') && errors.length > 0) {
      return true;
    } else if (outcome.contains('static type warning')
        && staticWarnings.length > 0) {
      return true;
    } else if (outcome.isEmpty
        && (errors.length > 0 || staticWarnings.length > 0)) {
      return true;
    }
    return false;
  }

  bool _didStandardTestFail(List errors, List staticWarnings) {
    bool hasFatalTypeErrors = false;
    int numStaticTypeAnnotations = 0;
    int numCompileTimeAnnotations = 0;
    var isStaticClean = false;
    if (testCase.info != null) {
      var optionsFromFile = testCase.info.optionsFromFile;
      hasFatalTypeErrors = testCase.info.hasFatalTypeErrors;
      for (Command c in testCase.commands) {
        for (String arg in c.arguments) {
          if (arg == '--fatal-type-errors') {
            hasFatalTypeErrors = true;
            break;
          }
        }
      }
      numStaticTypeAnnotations = optionsFromFile['numStaticTypeAnnotations'];
      numCompileTimeAnnotations = optionsFromFile['numCompileTimeAnnotations'];
      isStaticClean = optionsFromFile['isStaticClean'];
    }

    if (errors.length == 0) {
      if (!hasFatalTypeErrors && exitCode != 0) {
        diagnostics.add("EXIT CODE MISMATCH: Expected error message:");
        diagnostics.add("  command[0]:${testCase.commands[0]}");
        diagnostics.add("  exitCode:${exitCode}");
        return true;
      }
    } else if (exitCode == 0) {
      diagnostics.add("EXIT CODE MISMATCH: Unexpected error message:");
      diagnostics.add("  errors[0]:${errors[0]}");
      diagnostics.add("  command[0]:${testCase.commands[0]}");
      diagnostics.add("  exitCode:${exitCode}");
      return true;
    }
    if (numStaticTypeAnnotations > 0 && isStaticClean) {
      diagnostics.add("Cannot have both @static-clean and /// static "
                      "type warning annotations.");
      return true;
    }

    if (isStaticClean && staticWarnings.length > 0) {
      diagnostics.add(
          "@static-clean annotation found but analyzer returned warnings.");
      return true;
    }

    if (numCompileTimeAnnotations > 0
        && numCompileTimeAnnotations < errors.length) {
      // Expected compile-time errors were not returned.
      // The test did not 'fail' in the way intended so don't return failed.
      diagnostics.add("Fewer compile time errors than annotated: "
          "$numCompileTimeAnnotations");
      return false;
    }

    if (numStaticTypeAnnotations > 0 || hasFatalTypeErrors) {
      // TODO(zundel): match up the annotation line numbers
      // with the reported error line numbers
      if (staticWarnings.length < numStaticTypeAnnotations) {
        diagnostics.add("Fewer static type warnings than annotated: "
            "$numStaticTypeAnnotations");
        return true;
      }
      return false;
    } else if (errors.length != 0) {
      return true;
    }
    return false;
  }

  // Parse a line delimited by the | character using \ as an escape charager
  // like:  FOO|BAR|FOO\|BAR|FOO\\BAZ as 4 fields: FOO BAR FOO|BAR FOO\BAZ
  List<String> splitMachineError(String line) {
    StringBuffer field = new StringBuffer();
    List<String> result = [];
    bool escaped = false;
    for (var i = 0 ; i < line.length; i++) {
      var c = line[i];
      if (!escaped && c == '\\') {
        escaped = true;
        continue;
      }
      escaped = false;
      if (c == '|') {
        result.add(field.toString());
        field.clear();
        continue;
      }
      field.add(c);
    }
    result.add(field.toString());
    return result;
  }
}

/**
 * A RunningProcess actually runs a test, getting the command lines from
 * its [TestCase], starting the test process (and first, a compilation
 * process if the TestCase is a [BrowserTestCase]), creating a timeout
 * timer, and recording the results in a new [CommandOutput] object, which it
 * attaches to the TestCase.  The lifetime of the RunningProcess is limited
 * to the time it takes to start the process, run the process, and record
 * the result; there are no pointers to it, so it should be available to
 * be garbage collected as soon as it is done.
 */
class RunningProcess {
  TestCase testCase;
  Command command;
  bool timedOut = false;
  Date startTime;
  Timer timeoutTimer;
  List<int> stdout = <int>[];
  List<int> stderr = <int>[];
  bool compilationSkipped = false;
  Completer<CommandOutput> completer;

  RunningProcess(TestCase this.testCase, Command this.command);

  Future<CommandOutput> start() {
    Expect.isFalse(testCase.expectedOutcomes.contains(SKIP));

    completer = new Completer<CommandOutput>();
    startTime = new Date.now();
    _runCommand();
    return completer.future;
  }

  void _runCommand() {
    command.outputIsUpToDate.then((bool isUpToDate) {
      if (isUpToDate) {
        compilationSkipped = true;
        _commandComplete(0);
      } else {
        var processOptions = _createProcessOptions();
        Future processFuture = io.Process.start(command.executable,
                                                command.arguments,
                                                processOptions);
        processFuture.then((io.Process process) {
          void timeoutHandler(_) {
            timedOut = true;
            if (process != null) {
              process.kill();
            }
          }
          process.onExit = _commandComplete;
          _drainStream(process.stdout, stdout);
          _drainStream(process.stderr, stderr);
          timeoutTimer = new Timer(1000 * testCase.timeout, timeoutHandler);
        }).catchError((e) {
          print("Process error:");
          print("  Command: $command");
          print("  Error: $e");
          _commandComplete(-1);
          return true;
        });
      }
    });
  }

  void _commandComplete(int exitCode) {
    if (timeoutTimer != null) {
      timeoutTimer.cancel();
    }
    var commandOutput = _createCommandOutput(command, exitCode);
    completer.complete(commandOutput);
  }

  CommandOutput _createCommandOutput(Command command, int exitCode) {
    var incomplete = command != testCase.commands.last;
    var commandOutput = new CommandOutput.fromCase(
        testCase,
        command,
        exitCode,
        incomplete,
        timedOut,
        stdout,
        stderr,
        new Date.now().difference(startTime),
        compilationSkipped);
    return commandOutput;
  }

  void _drainStream(io.InputStream source, List<int> destination) {
    void onDataHandler () {
      if (source.closed) {
        return;  // TODO(whesse): Remove when bug is fixed.
      }
      var data = source.read();
      while (data != null) {
        destination.addAll(data);
        data = source.read();
      }
    }
    source.onData = onDataHandler;
    source.onClosed = onDataHandler;
  }

  io.ProcessOptions _createProcessOptions() {
    var baseEnvironment = command.environment != null ?
        command.environment : io.Platform.environment;
    io.ProcessOptions options = new io.ProcessOptions();
    options.environment = new Map<String, String>.from(baseEnvironment);
    options.environment['DART_CONFIGURATION'] =
        TestUtils.configurationDir(testCase.configuration);
    return options;
  }
}

/**
 * This class holds a value, that can be changed.  It is used when
 * closures need a shared value, that they can all change and read.
 */
class MutableValue<T> {
  MutableValue(T this.value);
  T value;
}

class BatchRunnerProcess {
  Command _command;
  String _executable;
  List<String> _batchArguments;

  io.Process _process;
  io.StringInputStream _stdoutStream;
  io.StringInputStream _stderrStream;

  TestCase _currentTest;
  List<int> _testStdout;
  List<int> _testStderr;
  String _status;
  bool _stdoutDrained = false;
  bool _stderrDrained = false;
  MutableValue<bool> _ignoreStreams;
  Date _startTime;
  Timer _timer;

  bool _isWebDriver;

  BatchRunnerProcess(TestCase testCase) {
    _command = testCase.commands.last;
    _executable = testCase.commands.last.executable;
    _batchArguments = testCase.batchRunnerArguments;
    _isWebDriver = testCase.usesWebDriver;
  }

  bool get active => _currentTest != null;

  void startTest(TestCase testCase) {
    Expect.isNull(_currentTest);
    _currentTest = testCase;
    _command = testCase.commands.last;
    if (_process == null) {
      // Start process if not yet started.
      _executable = testCase.commands.last.executable;
      _startProcess(() {
        doStartTest(testCase);
      });
    } else if (testCase.commands.last.executable != _executable) {
      // Restart this runner with the right executable for this test
      // if needed.
      _executable = testCase.commands.last.executable;
      _batchArguments = testCase.batchRunnerArguments;
      _process.onExit = (exitCode) {
        _startProcess(() {
          doStartTest(testCase);
        });
      };
      _process.kill();
    } else {
      doStartTest(testCase);
    }
  }

  Future terminate() {
    if (_process == null) return new Future.immediate(true);
    Completer completer = new Completer();
    Timer killTimer;
    _process.onExit = (exitCode) {
      if (killTimer != null) killTimer.cancel();
      completer.complete(true);
    };
    if (_isWebDriver) {
      // Use a graceful shutdown so our Selenium script can close
      // the open browser processes. On Windows, signals do not exist
      // and a kill is a hard kill.
      _process.stdin.write('--terminate\n'.charCodes);

      // In case the run_selenium process didn't close, kill it after 30s
      int shutdownMillisecs = 30000;
      killTimer = new Timer(shutdownMillisecs, (e) { _process.kill(); });
    } else {
      _process.kill();
    }

    return completer.future;
  }

  void doStartTest(TestCase testCase) {
    _startTime = new Date.now();
    _testStdout = [];
    _testStderr = [];
    _status = null;
    _stdoutDrained = false;
    _stderrDrained = false;
    _ignoreStreams = new MutableValue<bool>(false);  // Captured by closures.
    _readStdout(_stdoutStream, _testStdout);
    _readStderr(_stderrStream, _testStderr);
    _timer = new Timer(testCase.timeout * 1000, _timeoutHandler);

    if (testCase.commands.last.environment != null) {
      print("Warning: command.environment != null, but we don't support custom "
            "environments for batch runner tests!");
    }

    var line = _createArgumentsLine(testCase.batchTestArguments);
    _process.stdin.onError = (err) {
      print('Error on batch runner input stream stdin');
      print('  Input line: $line');
      print('  Previous test\'s status: $_status');
      print('  Error: $err');
      throw err;
    };
    _process.stdin.write(line.charCodes);
  }

  String _createArgumentsLine(List<String> arguments) {
    return arguments.join(' ').concat('\n');
  }

  void _reportResult() {
    if (!active) return;
    // _status == '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}'

    var outcome = _status.split(" ")[2];
    var exitCode = 0;
    if (outcome == "CRASH") exitCode = CRASHING_BROWSER_EXITCODE;
    if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1;
    new CommandOutput.fromCase(_currentTest,
                               _command,
                               exitCode,
                               false,
                               (outcome == "TIMEOUT"),
                               _testStdout,
                               _testStderr,
                               new Date.now().difference(_startTime),
                               false);
    var test = _currentTest;
    _currentTest = null;
    test.completed();
  }

  void _stderrDone() {
    _stderrDrained = true;
    // Move on when both stdout and stderr has been drained.
    if (_stdoutDrained) _reportResult();
  }

  void _stdoutDone() {
    _stdoutDrained = true;
    // Move on when both stdout and stderr has been drained.
    if (_stderrDrained) _reportResult();
  }

  void _readStdout(io.StringInputStream stream, List<int> buffer) {
    var ignoreStreams = _ignoreStreams;  // Capture this mutable object.
    void onLineHandler() {
      if (ignoreStreams.value) {
         while (stream.readLine() != null) {
          // Do nothing.
        }
        return;
      }
      // Otherwise, process output and call _reportResult() when done.
      var line = stream.readLine();
      while (line != null) {
        if (line.startsWith('>>> TEST')) {
          _status = line;
        } else if (line.startsWith('>>> BATCH START')) {
          // ignore
        } else if (line.startsWith('>>> ')) {
          throw new Exception('Unexpected command from dartc batch runner.');
        } else {
          buffer.addAll(encodeUtf8(line));
          buffer.addAll("\n".charCodes);
        }
        line = stream.readLine();
      }
      if (_status != null) {
        _timer.cancel();
        _stdoutDone();
      }
    }
    stream.onLine = onLineHandler;
  }

  void _readStderr(io.StringInputStream stream, List<int> buffer) {
    var ignoreStreams = _ignoreStreams;  // Capture this mutable object.
    void onLineHandler() {
      if (ignoreStreams.value) {
        while (stream.readLine() != null) {
          // Do nothing.
        }
        return;
      }
      // Otherwise, process output and call _reportResult() when done.
      var line = stream.readLine();
      while (line != null) {
        if (line.startsWith('>>> EOF STDERR')) {
          _stderrDone();
        } else {
          buffer.addAll(encodeUtf8(line));
          buffer.addAll("\n".charCodes);
        }
        line = stream.readLine();
      }
    }
    stream.onLine = onLineHandler;
  }

  ExitCodeEvent makeExitHandler(String status) {
    void handler(int exitCode) {
      if (active) {
        if (_timer != null) _timer.cancel();
        _status = status;
        // Read current content of streams, ignore any later output.
        _ignoreStreams.value = true;
        var line = _stdoutStream.readLine();
        while (line != null) {
          _testStdout.add(line);
          line = _stdoutStream.readLine();
        }
        line = _stderrStream.readLine();
        while (line != null) {
          _testStderr.add(line);
          line = _stderrStream.readLine();
        }
        _stderrDrained = true;
        _stdoutDrained = true;
        _startProcess(_reportResult);
      } else {  // No active test case running.
        _process = null;
      }
    }
    return handler;
  }

  void _timeoutHandler(ignore) {
    _process.onExit = makeExitHandler(">>> TEST TIMEOUT");
    _process.kill();
  }

  _startProcess(callback) {
    Future processFuture = io.Process.start(_executable, _batchArguments);
    processFuture.then((io.Process p) {
      _process = p;
      _stdoutStream = new io.StringInputStream(_process.stdout);
      _stderrStream = new io.StringInputStream(_process.stderr);
      _process.onExit = makeExitHandler(">>> TEST CRASH");
      callback();
    }).catchError((e) {
      print("Process error:");
      print("  Command: $_executable ${_batchArguments.join(' ')}");
      print("  Error: $e");
      // If there is an error starting a batch process, chances are that
      // it will always fail. So rather than re-trying a 1000+ times, we
      // exit.
      exit(1);
      return true;
    });
  }
}

/**
 * ProcessQueue is the master control class, responsible for running all
 * the tests in all the TestSuites that have been registered.  It includes
 * a rate-limited queue to run a limited number of tests in parallel,
 * a ProgressIndicator which prints output when tests are started and
 * and completed, and a summary report when all tests are completed,
 * and counters to determine when all of the tests in all of the test suites
 * have completed.
 *
 * Because multiple configurations may be run on each test suite, the
 * ProcessQueue contains a cache in which a test suite may record information
 * about its list of tests, and may retrieve that information when it is called
 * upon to enqueue its tests again.
 */
class ProcessQueue {
  int _numProcesses = 0;
  int _maxProcesses;
  int _numBrowserProcesses = 0;
  int _maxBrowserProcesses;
  bool _allTestsWereEnqueued = false;

  /** The number of tests we allow to actually fail before we stop retrying. */
  int _MAX_FAILED_NO_RETRY = 4;
  bool _verbose;
  bool _listTests;
  Function _allDone;
  Queue<TestCase> _tests;
  ProgressIndicator _progress;

  // For dartc/selenium batch processing we keep a list of batch processes.
  Map<String, List<BatchRunnerProcess>> _batchProcesses;

  // Cache information about test cases per test suite. For multiple
  // configurations there is no need to repeatedly search the file
  // system, generate tests, and search test files for options.
  Map<String, List<TestInformation>> _testCache;

  /**
   * String indicating the browser used to run the tests. Empty if no browser
   * used.
   */
  String browserUsed = '';

  /**
   * Process running the selenium server .jar (only used for Safari and Opera
   * tests.)
   */
  io.Process _seleniumServer = null;

  /** True if we are in the process of starting the server. */
  bool _startingServer = false;

  /** True if we find that there is already a selenium jar running. */
  bool _seleniumAlreadyRunning = false;

  ProcessQueue(this._maxProcesses,
               this._maxBrowserProcesses,
               String progress,
               Date startTime,
               bool printTiming,
               testSuites,
               this._allDone,
               [bool verbose = false,
                bool listTests = false])
      : _verbose = verbose,
        _listTests = listTests,
        _tests = new Queue<TestCase>(),
        _progress = new ProgressIndicator.fromName(progress,
                                                   startTime,
                                                   printTiming),
        _batchProcesses = new Map<String, List<BatchRunnerProcess>>(),
        _testCache = new Map<String, List<TestInformation>>() {
    _runTests(testSuites);
  }

  /**
   * Perform any cleanup needed once all tests in a TestSuite have completed
   * and notify our progress indicator that we are done.
   */
  void _cleanupAndMarkDone() {
    _allDone();
    if (browserUsed != '' && _seleniumServer != null) {
      _seleniumServer.kill();
    }
    _progress.allDone();
  }

  void _checkDone() {
    if (_allTestsWereEnqueued && _tests.isEmpty && _numProcesses == 0) {
      _terminateBatchRunners().then((_) => _cleanupAndMarkDone());
    }
  }

  void _runTests(List<TestSuite> testSuites) {
    // FIXME: For some reason we cannot call this method on all test suites
    // in parallel.
    // If we do, not all tests get enqueued (if --arch=all was specified,
    // we don't get twice the number of tests [tested on -rvm -cnone])
    // Issue: 7927
    Iterator<TestSuite> iterator = testSuites.iterator;
    void enqueueNextSuite() {
      if (!iterator.moveNext()) {
        _allTestsWereEnqueued = true;
        _progress.allTestsKnown();
        _checkDone();
      } else {
        iterator.current.forEachTest(_runTest, _testCache, enqueueNextSuite);
      }
    }
    enqueueNextSuite();
  }

  /**
   * True if we are using a browser + platform combination that needs the
   * Selenium server jar.
   */
  bool get _needsSelenium => (io.Platform.operatingSystem == 'macos' &&
      browserUsed == 'safari') || browserUsed == 'opera';

  /** True if the Selenium Server is ready to be used. */
  bool get _isSeleniumAvailable => _seleniumServer != null ||
      _seleniumAlreadyRunning;

  /**
   * Restart all the processes that have been waiting/stopped for the server to
   * start up. If we just call this once we end up with a single-"threaded" run.
   */
  void resumeTesting() {
    for (int i = 0; i < _maxProcesses; i++) _tryRunTest();
  }

  /** Start the Selenium Server jar, if appropriate for this platform. */
  void _ensureSeleniumServerRunning() {
    if (!_isSeleniumAvailable && !_startingServer) {
      _startingServer = true;

      // Check to see if the jar was already running before the program started.
      String cmd = 'ps';
      var arg = ['aux'];
      if (io.Platform.operatingSystem == 'windows') {
        cmd = 'tasklist';
        arg.add('/v');
      }

      Future processFuture = io.Process.start(cmd, arg);
      processFuture.then((io.Process p) {
        // Drain stderr to not leak resources.
        p.stderr.onData = p.stderr.read;
        final io.StringInputStream stdoutStringStream =
            new io.StringInputStream(p.stdout);
        stdoutStringStream.onLine = () {
          var line = stdoutStringStream.readLine();
          while (null != line) {
            var regexp = new RegExp(r".*selenium-server-standalone.*");
            if (regexp.hasMatch(line)) {
              _seleniumAlreadyRunning = true;
              resumeTesting();
            }
            line = stdoutStringStream.readLine();
          }
          if (!_isSeleniumAvailable) {
            _startSeleniumServer();
          }
        };
      }).catchError((e) {
        print("Error starting process:");
        print("  Command: $cmd ${arg.join(' ')}");
        print("  Error: $e");
        // TODO(ahe): How to report this as a test failure?
        exit(1);
        return true;
      });
    }
  }

  void _runTest(TestCase test) {
    if (test.usesWebDriver) {
      browserUsed = test.configuration['browser'];
      if (_needsSelenium) _ensureSeleniumServerRunning();
    }
    _progress.testAdded();
    _tests.add(test);
    _tryRunTest();
  }

  /**
   * Monitor the output of the Selenium server, to know when we are ready to
   * begin running tests.
   * source: Output(Stream) from the Java server.
   */
  VoidFunction makeSeleniumServerHandler(io.StringInputStream source) {
    void handler() {
      if (source.closed) return;  // TODO(whesse): Remove when bug is fixed.
      var line = source.readLine();
      while (null != line) {
        if (new RegExp(r".*Started.*Server.*").hasMatch(line) ||
            new RegExp(r"Exception.*Selenium is already running.*").hasMatch(
            line)) {
          resumeTesting();
        }
        line = source.readLine();
      }
    }
    return handler;
  }

  /**
   * For browser tests using Safari or Opera, we need to use the Selenium 1.0
   * Java server.
   */
  void _startSeleniumServer() {
    // Get the absolute path to the Selenium jar.
    String filePath = TestUtils.testScriptPath;
    String pathSep = io.Platform.pathSeparator;
    int index = filePath.lastIndexOf(pathSep);
    filePath = '${filePath.substring(0, index)}${pathSep}testing${pathSep}';
    var lister = new io.Directory(filePath).list();
    lister.onFile = (String file) {
      if (new RegExp(r"selenium-server-standalone-.*\.jar").hasMatch(file)
          && _seleniumServer == null) {
        Future processFuture = io.Process.start('java', ['-jar', file]);
        processFuture.then((io.Process server) {
          _seleniumServer = server;
          // Heads up: there seems to an obscure data race of some form in
          // the VM between launching the server process and launching the test
          // tasks that disappears when you read IO (which is convenient, since
          // that is our condition for knowing that the server is ready).
          io.StringInputStream stdoutStringStream =
              new io.StringInputStream(_seleniumServer.stdout);
          io.StringInputStream stderrStringStream =
              new io.StringInputStream(_seleniumServer.stderr);
          stdoutStringStream.onLine =
              makeSeleniumServerHandler(stdoutStringStream);
          stderrStringStream.onLine =
              makeSeleniumServerHandler(stderrStringStream);
        }).catchError((e) {
          print("Process error:");
          print("  Command: java -jar $file");
          print("  Error: $e");
          // TODO(ahe): How to report this as a test failure?
          exit(1);
          return true;
        });
      }
    };
  }

  Future _terminateBatchRunners() {
    var futures = new List();
    for (var runners in _batchProcesses.values) {
      for (var runner in runners) {
        futures.add(runner.terminate());
      }
    }
    // Change to Future.wait when updating binaries.
    return Future.wait(futures);
  }

  BatchRunnerProcess _getBatchRunner(TestCase test) {
    // Start batch processes if needed
    var compiler = test.configuration['compiler'];
    var runners = _batchProcesses[compiler];
    if (runners == null) {
      runners = new List<BatchRunnerProcess>(_maxProcesses);
      for (int i = 0; i < _maxProcesses; i++) {
        runners[i] = new BatchRunnerProcess(test);
      }
      _batchProcesses[compiler] = runners;
    }

    for (var runner in runners) {
      if (!runner.active) return runner;
    }
    throw new Exception('Unable to find inactive batch runner.');
  }

  void _tryRunTest() {
    _checkDone();
    if (_numProcesses < _maxProcesses && !_tests.isEmpty) {
      TestCase test = _tests.removeFirst();
      if (_listTests) {
        var fields = [test.displayName,
                      test.expectedOutcomes.join(','),
                      test.isNegative.toString()];
        fields.addAll(test.commands.last.arguments);
        print(fields.join('\t'));
        return;
      }
      if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable || (test
          is BrowserTestCase && test.waitingForOtherTest)) {
        // The test is not yet ready to run. Put the test back in
        // the queue.  Avoid spin-polling by using a timeout.
        _tests.add(test);
        new Timer(100, (_) => _tryRunTest());  // Don't lose a process.
        return;
      }
      if (_verbose) {
        int i = 1;
        if (test is BrowserTestCase) {
          // Additional command for rerunning the steps locally after the fact.
          print('$i. ${TestUtils.dartTestExecutable.toNativePath()} '
                '${TestUtils.dartDir().toNativePath()}/tools/testing/dart/'
                'http_server.dart -m ${test.configuration["mode"]} '
                '-a ${test.configuration["arch"]} '
                '-p ${http_server.TestingServerRunner.serverList[0].port} '
                '-c ${http_server.TestingServerRunner.serverList[1].port} '
                '--package-root='
                '${http_server.TestingServerRunner.packageRootDir}');
          i++;
        }
        for (Command command in test.commands) {
          print('$i. ${command.commandLine}');
          i++;
        }
      }

      var isLastCommand =
          ((test.commands.length-1) == test.commandOutputs.length);
      var isBrowserCommand = isLastCommand && (test is BrowserTestCase);
      if (isBrowserCommand && _numBrowserProcesses == _maxBrowserProcesses) {
        // If there is no free browser runner, put it back into the queue.
        _tests.add(test);
        new Timer(100, (_) => _tryRunTest());  // Don't lose a process.
        return;
      }

      _progress.start(test);

      // Dartc and browser test commands can be run by a [BatchRunnerProcess]
      var nextCommandIndex = test.commandOutputs.keys.length;
      var numberOfCommands = test.commands.length;
      var useBatchRunnerForDartc = test.configuration['compiler'] == 'dartc' &&
                                   test.displayName != 'dartc/junit_tests';
      var isWebdriverCommand = nextCommandIndex == (numberOfCommands - 1) &&
                               test.usesWebDriver &&
                               !test.configuration['noBatch'];
      if (useBatchRunnerForDartc || isWebdriverCommand) {
        TestCaseEvent oldCallback = test.completedHandler;
        void testCompleted(TestCase test_arg) {
          _numProcesses--;
          if (isBrowserCommand) {
            _numBrowserProcesses--;
          }
          _progress.done(test_arg);
          if (test_arg is BrowserTestCase) test_arg.notifyObservers();
          oldCallback(test_arg);
          _tryRunTest();
        };
        test.completedHandler = testCompleted;
        _getBatchRunner(test).startTest(test);
      } else {
        // Once we've actually failed a test, technically, we wouldn't need to
        // bother retrying any subsequent tests since the bot is already red.
        // However, we continue to retry tests until we have actually failed
        // four tests (arbitrarily chosen) for more debugable output, so that
        // the developer doesn't waste his or her time trying to fix a bunch of
        // tests that appear to be broken but were actually just flakes that
        // didn't get retried because there had already been one failure.
        bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
        runNextCommandWithRetries(test, allowRetry).then((TestCase testCase) {
          _numProcesses--;
          if (isBrowserCommand) {
            _numBrowserProcesses--;
          }
          if (isTestCaseFinished(testCase)) {
            testCase.completed();
            _progress.done(testCase);
            if (testCase is BrowserTestCase) testCase.notifyObservers();
          } else {
            _tests.addFirst(testCase);
          }
          _tryRunTest();
        });
      }

      _numProcesses++;
      if (isBrowserCommand) {
        _numBrowserProcesses++;
      }
    }
  }

  bool isTestCaseFinished(TestCase testCase) {
    var numberOfCommandOutputs = testCase.commandOutputs.keys.length;
    var numberOfCommands = testCase.commands.length;

    var lastCommandCompleted = (numberOfCommandOutputs == numberOfCommands);
    var lastCommandOutput = testCase.lastCommandOutput;
    var lastCommand = lastCommandOutput.command;
    var timedOut = lastCommandOutput.hasTimedOut;
    var nonZeroExitCode = lastCommandOutput.exitCode != 0;
    // NOTE: If this was the last command or there was unexpected output
    // we're done with the test.
    // Otherwise we need to enqueue it again into the test queue.
    if (lastCommandCompleted || timedOut || nonZeroExitCode) {
      var verbose = testCase.configuration['verbose'];
      if (lastCommandOutput.unexpectedOutput && verbose != null && verbose) {
        print(testCase.displayName);
        print("stderr:");
        print(decodeUtf8(lastCommandOutput.stderr));
        if (!lastCommand.isPixelTest) {
          print("stdout:");
          print(decodeUtf8(lastCommandOutput.stdout));
        } else {
          print("");
          print("DRT pixel test failed! stdout is not printed because it "
                "contains binary data!");
        }
      }
      return true;
    } else {
      return false;
    }
  }

  Future runNextCommandWithRetries(TestCase testCase, bool allowRetry) {
    var completer = new Completer();

    var nextCommandIndex = testCase.commandOutputs.keys.length;
    var numberOfCommands = testCase.commands.length;
    Expect.isTrue(nextCommandIndex < numberOfCommands);
    var command = testCase.commands[nextCommandIndex];
    var isLastCommand = nextCommandIndex == (numberOfCommands - 1);

    void runCommand() {
      var runningProcess = new RunningProcess(testCase, command);
      runningProcess.start().then((CommandOutput commandOutput) {
        if (isLastCommand) {
          // NOTE: We need to call commandOutput.unexpectedOutput here.
          // Calling this getter may result in the side-effect, that
          // commandOutput.requestRetry is set to true.
          // (BrowserCommandOutputImpl._failedBecauseOfMissingXDisplay
          // does that for example)
          // TODO(ricow/kustermann): Issue 8206
          var unexpectedOutput = commandOutput.unexpectedOutput;
          if (allowRetry && testCase.usesWebDriver
              && unexpectedOutput
              && (testCase as BrowserTestCase).numRetries > 0) {
            // Selenium tests can be flaky. Try rerunning.
            commandOutput.requestRetry = true;
          }
        }
        if (commandOutput.requestRetry) {
          commandOutput.requestRetry = false;
          (testCase as BrowserTestCase).numRetries--;
          DebugLogger.warning("Rerunning Test: ${testCase.displayName} "
                              "(${(testCase as BrowserTestCase).numRetries} "
                              "attempt(s) remains) [cmd:$command]");
          runCommand();
        } else {
          completer.complete(testCase);
        }
      });
    }
    runCommand();

    return completer.future;
  }
}