runtime/tools/dartfuzz/run_dartfuzz_test.py

#!/usr/bin/env python3
#
# Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code == governed by a
# BSD-style license that can be found in the LICENSE file.

import abc
import argparse
import os
import shutil
import signal
import subprocess
import sys

from enum import Enum
from enum import unique

from subprocess import DEVNULL
from subprocess import PIPE
from subprocess import Popen
from subprocess import STDOUT
from subprocess import TimeoutExpired

from tempfile import mkdtemp

#
# Helper methods to run commands.
#

@unique
class RetCode(Enum):
  """Enum representing return codes."""
  SUCCESS = 0
  TIMEOUT = 1
  ERROR = 2

class FatalError(Exception):
  """Fatal error in script."""

def RunCommandWithOutput(cmd, env, stdout, stderr, timeout=30):
  """Runs command piping output to files, stderr, or stdout.

  Args:
    cmd: list of strings, command to run.
    env: shell environment for command.
    stdout: file handle for stdout.
    stderr: file handle for stderr.
    timeout: int, timeout in seconds.

  Returns:
    tuple (string, string, RetCode) out, err, return code.
  """
  proc = Popen(cmd, stdout=stdout, stderr=stderr, env=env,
               universal_newlines=True, start_new_session=True)
  try:
    (out, err) = proc.communicate(timeout=timeout)
    if proc.returncode == 0:
      retcode = RetCode.SUCCESS
    else:
      retcode = RetCode.ERROR
  except TimeoutExpired:
    os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
    (out, err) = proc.communicate()
    retcode = RetCode.TIMEOUT
  return (out, err, retcode)

def RunCommand(cmd, out=None, err=None, timeout=30):
  """Executes a command, and returns its return code.

  Args:
    cmd: list of strings, a command to execute.
    out: string, file name to open for stdout (or None).
    err: string, file name to open for stderr (or None).
    timeout: int, time out in seconds.
  Returns:
    RetCode, return code of running command.
  """
  if out is not None:
    outf = open(out, mode='w')
  else:
    outf = DEVNULL
  if err is not None:
    errf = open(err, mode='w')
  else:
    errf = DEVNULL
  (_, _, retcode) = RunCommandWithOutput(cmd, None, outf, errf, timeout)
  if outf != DEVNULL:
    outf.close()
  if errf != DEVNULL:
    errf.close()
  return retcode

#
# Execution modes.
#

class TestRunner(object):
  """Abstraction for running a test in a particular execution mode."""
  __meta_class__ = abc.ABCMeta

  @abc.abstractproperty
  def description(self):
    """Returns a description string of the execution mode."""

  @abc.abstractmethod
  def RunTest(self):
    """Run the generated test.

    Ensures that the current fuzz.dart in the temporary directory is executed
    under the current execution mode.

    Most nonzero return codes are assumed non-divergent, since systems may
    exit in different ways. This is enforced by normalizing return codes.

    Returns:
      tuple (string, string, RetCode) stdout-output, stderr-output, return code.
    """

class TestRunnerDartJIT(TestRunner):
  """Concrete test runner of Dart JIT."""

  def  __init__(self, mode):
    self._mode = mode
    if mode == 'jit-ia32':
      self.AddPathAndTimeout('ReleaseIA32')
    elif mode == 'jit-x64':
      self.AddPathAndTimeout('ReleaseX64')
    elif mode == 'jit-arm32':
      self.AddPathAndTimeout('ReleaseSIMARM')
    elif mode == 'jit-arm64':
      self.AddPathAndTimeout('ReleaseSIMARM64')
    else:
      raise FatalError('Unknown JIT execution mode: ' + mode)

  @property
  def description(self):
    return 'Dart ' + self._mode

  def AddPathAndTimeout(self, tag, timeout=30):
    self._env = os.environ.copy()
    self._env['PATH'] = self._env['DART_TOP'] + '/out/' + tag + ':' + self._env['PATH']
    self._timeout = timeout

  def RunTest(self):
    return RunCommandWithOutput(
        ['dart', 'fuzz.dart'], self._env, PIPE, STDOUT, self._timeout)

class TestRunnerDartAOT(TestRunner):
  """Concrete test runner of Dart AOT."""

  def  __init__(self, mode):
    self._mode = mode
    if mode == 'aot-x64':
      self.AddPathConfigAndTimeout('ReleaseX64')
    elif mode == 'aot-arm64':
      self.AddPathConfigAndTimeout('ReleaseSIMARM64', 30 * 60)
    else:
      raise FatalError('Unknown AOT execution mode: ' + mode)

  @property
  def description(self):
    return 'Dart ' + self._mode

  def AddPathConfigAndTimeout(self, tag, timeout=30):
    self._env = os.environ.copy()
    self._env['PATH'] = self._env['DART_TOP'] + '/pkg/vm/tool:' + self._env['PATH']
    self._env['DART_CONFIGURATION'] = tag
    self._timeout = timeout

  def RunTest(self):
    (out, err, retcode) = RunCommandWithOutput(
        ['precompiler2', 'fuzz.dart', 'snap'],
        self._env, PIPE, STDOUT, self._timeout)
    if retcode != RetCode.SUCCESS:
      return (out, err, retcode)
    return RunCommandWithOutput(
        ['dart_precompiled_runtime2', 'snap'],
        self._env, PIPE, STDOUT, self._timeout)

class TestRunnerDart2JS(TestRunner):
  """Concrete test runner of Dart through dart2js and JS."""

  def  __init__(self):
    self.AddPath()

  @property
  def description(self):
    return 'Dart as JS'

  def AddPath(self):
    self._env = os.environ.copy()
    self._env['PATH'] = self._env['DART_TOP'] + '/ReleaseX64/dart-sdk/bin:' + self._env['PATH']

  def RunTest(self):
    (out, err, retcode) = RunCommandWithOutput(
        ['dart2js', 'fuzz.dart'], self._env, PIPE, STDOUT)
    if retcode != RetCode.SUCCESS:
      return (out, err, retcode)
    return RunCommandWithOutput(
        ['nodejs', 'out.js'], self._env, PIPE, STDOUT)

def GetExecutionModeRunner(mode):
  """Returns a runner for the given execution mode.

  Args:
    mode: string, execution mode
  Returns:
    TestRunner with given execution mode
  Raises:
    FatalError: error for unknown execution mode
  """
  if mode.startswith('jit'):
    return TestRunnerDartJIT(mode)
  if mode.startswith('aot'):
    return TestRunnerDartAOT(mode)
  if mode == 'js':
    return TestRunnerDart2JS()
  raise FatalError('Unknown execution mode: ' + mode)

#
# DartFuzzTester class.
#

class DartFuzzTester(object):
  """Tester that runs DartFuzz many times and report divergences."""

  def  __init__(self, repeat, true_divergence, mode1, mode2):
    """Constructor for the tester.

    Args:
      repeat: int, number of tests to run.
      true_divergence: boolean, report true divergences only.
      mode1: string, execution mode for first runner.
      mode2: string, execution mode for second runner.
    """
    self._repeat = repeat
    self._true_divergence = true_divergence
    self._runner1 = GetExecutionModeRunner(mode1)
    self._runner2 = GetExecutionModeRunner(mode2)

  def __enter__(self):
    """On entry, enters new temp directory after saving current directory.

    Raises:
      FatalError: error when temp directory cannot be constructed.
    """
    self._save_dir = os.getcwd()
    self._tmp_dir = mkdtemp(dir='/tmp/')
    if self._tmp_dir == None:
      raise FatalError('Cannot obtain temp directory')
    os.chdir(self._tmp_dir)
    return self

  def __exit__(self, etype, evalue, etraceback):
    """On exit, re-enters previously saved current directory and cleans up."""
    os.chdir(self._save_dir)
    if self._num_divergences == 0:
      shutil.rmtree(self._tmp_dir)
      print('\n\nsuccess (no divergences)\n')
    else:
      print('\n\nfailure (divergences):', self._tmp_dir, '\n')

  def Run(self):
    """Runs DartFuzz many times and report divergences."""
    self.Setup()
    print()
    print('**\n**** Dart Fuzz Testing\n**')
    print()
    print('#Tests      :', self._repeat)
    print('Exec-Mode 1 :', self._runner1.description)
    print('Exec-Mode 2 :', self._runner2.description)
    print('Dart Dev    :', os.environ.get('DART_TOP'))
    print('Orig Dir    :', self._save_dir)
    print('Temp Dir    :', self._tmp_dir)
    print()
    self.ShowStats()  # show all zeros on start
    for self._test in range(1, self._repeat + 1):
      self.RunTest()
      self.ShowStats()

  def Setup(self):
    """Initial setup of the testing environment."""
    # Fuzzer command.
    self._dartfuzz = self._save_dir + '/dartfuzz.dart'
    # Statistics.
    self._test = 0
    self._num_success = 0
    self._num_not_run = 0
    self._num_timed_out = 0
    self._num_divergences = 0

  def ShowStats(self):
    """Shows current statistics (on same line) while tester is running."""
    print('\rTests:', self._test,
          'Success:', self._num_success,
          'Not-run:', self._num_not_run,
          'Timed-out:', self._num_timed_out,
          'Divergences:', self._num_divergences,
          end='')
    sys.stdout.flush()

  def RunTest(self):
    """Runs a single fuzz test, comparing two execution modes."""
    self.ConstructTest()
    (out1, _, retcode1) = self._runner1.RunTest()
    (out2, _, retcode2) = self._runner2.RunTest()
    self.CheckForDivergence(out1, retcode1, out2, retcode2)
    self.CleanupTest()

  def ConstructTest(self):
    """Use DartFuzz to generate next fuzz.dart test.

    Raises:
      FatalError: error when DartFuzz fails.
    """
    # Invoke dartfuzz script on command line rather than calling py code.
    if (RunCommand(['dart', self._dartfuzz], out='fuzz.dart') != RetCode.SUCCESS):
      raise FatalError('Unexpected error while running DartFuzz')

  def CheckForDivergence(self, out1, retcode1, out2, retcode2):
    """Checks for divergences and updates statistics.

    Args:
      out1: string, output for first runner.
      retcode1: int, normalized return code of first runner.
      out2: string, output for second runner.
      retcode2: int, normalized return code of second runner.
    """
    if retcode1 == retcode2:
      # No divergence in return code.
      if retcode1 == RetCode.SUCCESS:
        # Both compilations and runs were successful, inspect generated output.
        if out1 == out2:
          # No divergence in output.
          self._num_success += 1
        else:
          # Divergence in output.
          self.ReportDivergence(out1, retcode1, out2, retcode2, True)
      elif retcode1 == RetCode.ERROR:
        # Both did not run.
        self._num_not_run += 1
      elif retcode1 == RetCode.TIMEOUT:
        # Both timed out.
        self._num_timed_out += 1
      else:
       raise FatalError('Unknown return code')
    else:
      # Divergence in return code.
      if self._true_divergence:
        # When only true divergences are requested, any divergence in return
        # code where one is a time out is treated as a regular time out.
        if RetCode.TIMEOUT in (retcode1, retcode2):
          self._num_timed_out += 1
          return
      self.ReportDivergence(out1, retcode1, out2, retcode2, False)

  def ReportDivergence(self, out1, retcode1, out2, retcode2, is_output_divergence):
    """Reports and saves a divergence.

    Args:
      out1: string, output for first runner.
      retcode1: int, normalized return code of first runner.
      out2: string, output for second runner.
      retcode2: int, normalized return code of second runner.
      is_output_divergence, boolean, denotes output divergence.
      """
    self._num_divergences += 1
    print('\n#' + str(self._num_divergences), end='')
    if is_output_divergence:
      print(' divergence in output')
    else:
      print(' divergence in return code: '
            + retcode1.name + ' vs. ' + retcode2.name)
    print('->')
    print(out1, end='')
    print('<-')
    print(out2, end='')
    print('--')
    # Save.
    ddir = self._tmp_dir + '/divergence' + str(self._num_divergences)
    os.mkdir(ddir)
    shutil.copy('fuzz.dart', ddir)
    # TODO: file bug report

  def CleanupTest(self):
    """Cleans up after a single test run."""
    for file_name in os.listdir(self._tmp_dir):
      file_path = os.path.join(self._tmp_dir, file_name)
      if os.path.isfile(file_path):
        os.unlink(file_path)
      elif os.path.isdir(file_path):
        pass  # keep the divergences directories

#
# Main driver.
#

def main():
  # Handle arguments.
  parser = argparse.ArgumentParser()
  parser.add_argument('--repeat', default=1000, type=int,
                      help='number of tests to run (default: 1000)')
  parser.add_argument('--true_divergence', default=False, action='store_true',
                      help='only report true divergences')
  parser.add_argument('--mode1', default='jit-x64',
                      help='execution mode 1 (default: jit-x64)')
  parser.add_argument('--mode2', default='aot-x64',
                      help='execution mode 2 (default: aot-x64)')
  args = parser.parse_args()

  # Verify DART_TOP.
  if os.environ.get('DART_TOP') == None:
    raise FatalError('DART_TOP needs to point to the Dart SDK tree')

  # Run DartFuzz tester.
  with DartFuzzTester(args.repeat,
                      args.true_divergence,
                      args.mode1,
                      args.mode2) as fuzzer:
    fuzzer.Run()

if __name__ == '__main__':
  main()