pkg/test_runner/lib/src/test_case.dart

// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
// We need to use the 'io' prefix here, otherwise io.exitCode will shadow
// CommandOutput.exitCode in subclasses of CommandOutput.
import 'dart:io' as io;

import "package:status_file/expectation.dart";

import 'command.dart';
import 'command_output.dart';
import 'configuration.dart';
import 'options.dart';
import 'output_log.dart';
import 'process_queue.dart';
import 'test_file.dart';
import 'utils.dart';

const _slowTimeoutMultiplier = 4;
const _extraSlowTimeoutMultiplier = 8;
const nonUtfFakeExitCode = 0xfffd;
const truncatedFakeExitCode = 0xfffc;

/// Some IO tests use these variables and get confused if the host environment
/// variables are inherited so they are excluded.
const _excludedEnvironmentVariables = [
  'http_proxy',
  'https_proxy',
  'no_proxy',
  'HTTP_PROXY',
  'HTTPS_PROXY',
  'NO_PROXY',
];

/// TestCase contains all the information needed to run a test and evaluate
/// its output.  Running a test involves starting a separate process, with
/// the executable and arguments given by the TestCase, and recording its
/// stdout and stderr output streams, and its exit code.  TestCase only
/// contains static information about the test; actually running the test is
/// performed by [ProcessQueue] using a [RunningProcess] object.
///
/// The output information is stored in a [CommandOutput] instance contained
/// in TestCase.commandOutputs. The last CommandOutput instance is responsible
/// for evaluating if the test has passed, failed, crashed, or timed out, and
/// the TestCase has information about what the expected result of the test
/// should be.
class TestCase {
  /// A list of commands to execute. Most test cases have a single command.
  /// Dart2js tests have two commands, one to compile the source and another
  /// to execute it. Some isolate tests might even have three, if they require
  /// compiling multiple sources that are run in isolation.
  List<Command> commands;
  Map<Command, CommandOutput> commandOutputs = {};

  TestConfiguration configuration;
  String displayName;
  Set<Expectation> expectedOutcomes;
  final TestFile testFile;

  TestCase(
    this.displayName,
    this.commands,
    this.configuration,
    this.expectedOutcomes,
    this.testFile,
  ) {
    // A test case should do something.
    assert(commands.isNotEmpty);
  }

  List<String> get experiments => getExperiments(testFile, configuration);

  static List<String> getExperiments(
    TestFile testFile,
    TestConfiguration configuration,
  ) {
    return [...testFile.experiments, ...configuration.experiments];
  }

  TestCase indexedCopy(int index) {
    var newCommands = commands.map((c) => c.indexedCopy(index)).toList();
    return TestCase(
      displayName,
      newCommands,
      configuration,
      expectedOutcomes,
      testFile,
    );
  }

  bool get hasRuntimeError => testFile.hasRuntimeError;
  bool get hasStaticWarning => testFile.hasStaticWarning;
  bool get hasSyntaxError => testFile.hasSyntaxError;
  bool get hasCompileError => testFile.hasCompileError;
  bool get hasCrash => testFile.hasCrash;

  Expectation get result => lastCommandOutput.result(this);
  Expectation get realExpected {
    if (hasCrash) {
      return Expectation.crash;
    }
    if (configuration.compiler == Compiler.specParser) {
      if (hasSyntaxError) {
        return Expectation.syntaxError;
      }
    } else if (hasCompileError) {
      if (hasRuntimeError && configuration.runtime != Runtime.none) {
        return Expectation.fail;
      }
      return Expectation.compileTimeError;
    }
    if (hasRuntimeError) {
      if (configuration.runtime != Runtime.none) {
        return Expectation.runtimeError;
      }
      return Expectation.pass;
    }
    if (configuration.compiler == Compiler.dart2analyzer && hasStaticWarning) {
      return Expectation.staticWarning;
    }
    return Expectation.pass;
  }

  CommandOutput get lastCommandOutput {
    if (commandOutputs.isEmpty) {
      throw Exception(
        "CommandOutputs is empty, maybe no command was run? ("
        "displayName: '$displayName', "
        "configurationString: '$configurationString')",
      );
    }
    return commandOutputs[commands[commandOutputs.length - 1]]!;
  }

  Command get lastCommandExecuted {
    if (commandOutputs.isEmpty) {
      throw Exception(
        "CommandOutputs is empty, maybe no command was run? ("
        "displayName: '$displayName', "
        "configurationString: '$configurationString')",
      );
    }
    return commands[commandOutputs.length - 1];
  }

  int get timeout {
    var result = configuration.timeout(displayName);
    if (expectedOutcomes.contains(Expectation.slow)) {
      result *= _slowTimeoutMultiplier;
    } else if (expectedOutcomes.contains(Expectation.extraSlow)) {
      result *= _extraSlowTimeoutMultiplier;
    }
    return result;
  }

  int get repeat => configuration.repeat(displayName);

  String get configurationString {
    var compiler = configuration.compiler.name;
    var runtime = configuration.runtime.name;
    var mode = configuration.mode.name;
    var arch = configuration.architecture.name;
    var checked = configuration.isChecked ? '-checked' : '';
    return "$compiler-$runtime$checked ${mode}_$arch";
  }

  List<String> get batchTestArguments {
    assert(commands.last is ProcessCommand);
    return (commands.last as ProcessCommand).arguments;
  }

  bool get isFinished {
    return commandOutputs.isNotEmpty &&
        (!lastCommandOutput.successful ||
            commands.length == commandOutputs.length);
  }
}

/// Helper to get a list of all child pids for a parent process.
Future<List<int>> _getPidList(int parentId, List<String> diagnostics) async {
  var pids = [parentId];
  late List<String> lines;
  var startLine = 0;
  if (io.Platform.isLinux || io.Platform.isMacOS) {
    var result = await io.Process.run("pgrep", [
      "-P",
      "${pids[0]}",
    ], runInShell: true);
    lines = (result.stdout as String).split('\n');
  } else if (io.Platform.isWindows) {
    var result = await io.Process.run("wmic", [
      "process",
      "where",
      "(ParentProcessId=${pids[0]})",
      "get",
      "ProcessId",
    ], runInShell: true);
    lines = (result.stdout as String).split('\n');
    // Skip first line containing header "ProcessId".
    startLine = 1;
  } else {
    assert(false);
  }
  if (lines.length > startLine) {
    for (var i = startLine; i < lines.length; ++i) {
      var pid = int.tryParse(lines[i]);
      if (pid != null) pids.add(pid);
    }
  } else {
    diagnostics.add("Could not find child pids");
    diagnostics.addAll(lines);
  }
  return pids;
}

/// A RunningProcess actually runs a test, getting the command lines from
/// its [TestCase], starting the test process (and first, a compilation
/// process if the TestCase needs compilation), creating a timeout timer, and
/// recording the results in a new [CommandOutput] object, which it attaches to
/// the TestCase. The lifetime of the RunningProcess is limited to the time it
/// takes to start the process, run the process, and record the result. There
/// are no pointers to it, so it should be available to be garbage collected as
/// soon as it is done.
class RunningProcess {
  final ProcessCommand command;
  final int timeout;
  bool timedOut = false;
  late DateTime startTime;
  int? pid;
  final OutputLog _stdout;
  final OutputLog _stderr = OutputLog();
  final List<String> diagnostics = [];
  bool compilationSkipped = false;
  late Completer<CommandOutput> completer;
  final TestConfiguration configuration;

  RunningProcess(
    this.command,
    this.timeout, {
    required this.configuration,
    io.File? outputFile,
  }) : _stdout = outputFile != null ? FileOutputLog(outputFile) : OutputLog();

  Future<CommandOutput> run() {
    completer = Completer();
    startTime = DateTime.now();
    _runCommand();
    return completer.future;
  }

  void _runCommand() {
    if (command.outputIsUpToDate) {
      compilationSkipped = true;
      _commandComplete(0);
    } else {
      var processEnvironment = _createProcessEnvironment();
      var args = [...command.nonBatchArguments, ...command.arguments];
      var processFuture = startProcess(
        command.executable,
        args,
        environment: processEnvironment,
        workingDirectory: command.workingDirectory,
      );
      processFuture
          .then<dynamic>((io.Process process) {
            var stdoutFuture = process.stdout.pipe(_stdout);
            var stderrFuture = process.stderr.pipe(_stderr);
            pid = process.pid;

            // Close stdin so that tests that try to block on input will fail.
            process.stdin.close();
            FutureOr<int> timeoutHandler() async {
              timedOut = true;
              String? executable;
              if (io.Platform.isLinux) {
                executable = 'eu-stack';
              } else if (io.Platform.isMacOS) {
                // Try to print stack traces of the timed out process.
                // `sample` is a sampling profiler but we ask it sample for 1
                // second with a 4 second delay between samples so that we only
                // sample the threads once.
                executable = '/usr/bin/sample';
              } else if (io.Platform.isWindows) {
                var isX64 =
                    command.executable.contains("X64") ||
                    command.executable.contains("SIMARM64") ||
                    command.executable.contains("SIMARM64C") ||
                    command.executable.contains("SIMRISCV64");
                var isARM64 = !isX64 && command.executable.contains("ARM64");
                if (configuration.windowsSdkPath != null) {
                  executable = [
                    configuration.windowsSdkPath!,
                    'Debuggers',
                    if (isX64) 'x64' else if (isARM64) 'arm64' else 'x86',
                    'cdb.exe',
                  ].join('\\');
                  diagnostics.add("Using $executable to print stack traces");
                } else {
                  diagnostics.add("win_sdk_path not found");
                }
              } else {
                diagnostics.add(
                  "Capturing stack traces on"
                  "${io.Platform.operatingSystem} not supported",
                );
              }
              if (executable != null) {
                var pids = await _getPidList(process.pid, diagnostics);
                diagnostics.add("Process list including children: $pids");
                for (var pid in pids) {
                  late List<String> arguments;
                  if (io.Platform.isLinux) {
                    arguments = ['-p $pid'];
                  } else if (io.Platform.isMacOS) {
                    arguments = ['$pid', '1', '4000', '-mayDie'];
                  } else if (io.Platform.isWindows) {
                    arguments = ['-p', '$pid', '-c', '!uniqstack;qd'];
                  } else {
                    assert(false);
                  }
                  diagnostics.add("Trying to capture stack trace for pid $pid");
                  try {
                    var result = await io.Process.run(executable, arguments);
                    diagnostics.addAll((result.stdout as String).split('\n'));
                    diagnostics.addAll((result.stderr as String).split('\n'));
                  } catch (error) {
                    diagnostics.add("Unable to capture stack traces: $error");
                  }
                }
              }
              if (!process.kill()) {
                diagnostics.add("Unable to kill ${process.pid}");
              }
              return 1;
            }

            // Wait for the process to finish or timeout.
            process.exitCode
                .timeout(Duration(seconds: timeout), onTimeout: timeoutHandler)
                .then((exitCode) {
                  // This timeout is used to close stdio to the subprocess once we got
                  // the exitCode. Sometimes descendants of the subprocess keep stdio
                  // handles alive even though the direct subprocess is dead.
                  Future.wait([stdoutFuture, stderrFuture])
                      .timeout(
                        maxStdioDelay,
                        onTimeout: () async {
                          DebugLogger.warning(
                            "$maxStdioDelayPassedMessage (command: $command)",
                          );
                          await _stdout.cancel();
                          await _stderr.cancel();
                          return [];
                        },
                      )
                      .then((_) {
                        if (_stdout is FileOutputLog) {
                          // Prevent logging data that has already been written to a file
                          // and is unlikely to add value in the logs because the command
                          // succeeded.
                          _stdout.clear();
                        }
                        _commandComplete(exitCode);
                      });
                });
          })
          .catchError((e) {
            // TODO(floitsch): should we try to report the stacktrace?
            print("Process error:");
            print("  Command: $command");
            print("  Executable: ${command.executable}");
            print("  Working directory: ${command.workingDirectory}");
            print("  Error: $e");
            _commandComplete(-1);
            return true;
          });
    }
  }

  void _commandComplete(int exitCode) {
    var commandOutput = _createCommandOutput(command, exitCode);
    completer.complete(commandOutput);
  }

  CommandOutput _createCommandOutput(ProcessCommand command, int exitCode) {
    var stdoutData = _stdout.bytes;
    var stderrData = _stderr.bytes;

    // Fail if the output was too long or incorrectly formatted.
    if (_stdout.hasNonUtf8 || _stderr.hasNonUtf8) {
      exitCode = nonUtfFakeExitCode;
    } else if (_stdout.wasTruncated || _stderr.wasTruncated) {
      exitCode = truncatedFakeExitCode;
    }

    var commandOutput = command.createOutput(
      exitCode,
      timedOut,
      stdoutData,
      stderrData,
      DateTime.now().difference(startTime),
      compilationSkipped,
      pid ?? 0,
    );
    commandOutput.diagnostics.addAll(diagnostics);
    return commandOutput;
  }

  Map<String, String> _createProcessEnvironment() {
    final environment = Map<String, String>.from(io.Platform.environment);
    environment.addAll(configuration.nativeCompilerEnvironmentVariables);
    environment.addAll(sanitizerEnvironmentVariables);
    for (var entry in command.environmentOverrides.entries) {
      environment[entry.key] = entry.value;
    }
    for (var excludedEnvironmentVariable in _excludedEnvironmentVariables) {
      environment.remove(excludedEnvironmentVariable);
    }
    if (configuration.keepGeneratedFiles) {
      // Used by withTempDir in many //runtime/tests/vm/dart tests.
      environment["KEEP_TEMPORARY_DIRECTORIES"] = "1";
    }
    return environment;
  }
}