tools/testing/dart/multitest.dart

// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// Multitests are Dart test scripts containing lines of the form
/// " [some dart code] //# [key]: [error type]"
///
/// To support legacy multi tests we also handle lines of the form
/// " [some dart code] /// [key]: [error type]"
///
/// For each key in the file, a new test file is made containing all the normal
/// lines of the file, and all of the multitest lines containing that key, in
/// the same order as in the source file. The new test is expected to pass if
/// the error type listed is 'ok', and to fail if the error type is 'syntax
/// error', 'compile-time error', 'runtime error', 'static type warning',
/// 'dynamic type error', or 'checked mode compile-time error'. The type error
/// tests fail only in checked mode. There is also a test created from only the
/// untagged lines of the file, with key "none", which is expected to pass. This
/// library extracts these tests, writes them into a temporary directory, and
/// passes them to the test runner. These tests may be referred to in the status
/// files with the pattern [test name]/[key].
///
/// For example, file i_am_a_multitest.dart:
///
/// ```dart
/// aaa
/// bbb //# 02: runtime error
/// ccc //# 02: continued
/// ddd //# 07: static type warning
/// eee //# 10: ok
/// fff
/// ```
///
/// Create four test files:
///
/// i_am_a_multitest_none.dart:
///
/// ```dart
/// aaa
/// fff
/// ```
///
/// i_am_a_multitest_02.dart:
///
/// ```dart
/// aaa
/// bbb //# 02: runtime error
/// ccc //# 02: continued
/// fff
/// ```
///
/// i_am_a_multitest_07.dart:
///
/// ```dart
/// aaa
/// ddd //# 07: static type warning
/// fff
/// ```
///
/// i_am_a_multitest_10.dart:
///
/// ```dart
/// aaa
/// eee //# 10: ok
/// fff
/// ```
///
/// Note that it is possible to indicate more than one acceptable outcome in
/// the case of dynamic and static type warnings
///
/// ```dart
/// aaa
/// ddd //# 07: static type warning, dynamic type error
/// fff
/// ```
import "dart:async";
import "dart:io";

import "path.dart";
import "test_suite.dart";
import "utils.dart";

/// Until legacy multitests are ported we need to support both /// and //#
final _multitestMarker = new RegExp(r"//[/#]");

final _multitestOutcomes = [
  'ok',
  'syntax error',
  'compile-time error',
  'runtime error',
  // TODO(rnystrom): Remove these after Dart 1.0 tests are removed.
  'static type warning',
  'dynamic type error',
  'checked mode compile-time error'
].toSet();

// Note: This function is called directly by:
//
//     tests/compiler/dart2js/frontend_checker.dart
//     tools/status_clean.dart
void extractTestsFromMultitest(Path filePath, Map<String, String> tests,
    Map<String, Set<String>> outcomes) {
  var contents = new File(filePath.toNativePath()).readAsStringSync();

  var firstNewline = contents.indexOf('\n');
  var lineSeparator =
      (firstNewline == 0 || contents[firstNewline - 1] != '\r') ? '\n' : '\r\n';
  var lines = contents.split(lineSeparator);
  if (lines.last == '') lines.removeLast();

  // Create the set of multitests, which will have a new test added each
  // time we see a multitest line with a new key.
  var testsAsLines = <String, List<String>>{};

  // Add the default case with key "none".
  testsAsLines['none'] = <String>[];
  outcomes['none'] = new Set<String>();

  var lineCount = 0;
  for (var line in lines) {
    lineCount++;
    var annotation = _Annotation.tryParse(line);
    if (annotation != null) {
      testsAsLines.putIfAbsent(
          annotation.key, () => new List<String>.from(testsAsLines["none"]));
      // Add line to test with annotation.key as key, empty line to the rest.
      for (var key in testsAsLines.keys) {
        testsAsLines[key].add(annotation.key == key ? line : "");
      }
      outcomes.putIfAbsent(annotation.key, () => new Set<String>());
      if (annotation.rest != 'continued') {
        for (var nextOutcome in annotation.outcomes) {
          if (_multitestOutcomes.contains(nextOutcome)) {
            outcomes[annotation.key].add(nextOutcome);
          } else {
            DebugLogger.warning(
                "Warning: Invalid expectation '$nextOutcome' on line "
                "$lineCount:\n${annotation.rest} ");
          }
        }
      }
    } else {
      for (var test in testsAsLines.values) test.add(line);
    }
  }

  // End marker, has a final line separator so we don't need to add it after
  // joining the lines.
  var marker =
      '// Test created from multitest named ${filePath.toNativePath()}.'
      '$lineSeparator';
  for (var test in testsAsLines.values) test.add(marker);

  // Check that every test (other than the none case) has at least one outcome.
  var invalidTests = outcomes.keys
      .where((test) => test != 'none' && outcomes[test].isEmpty)
      .toList();
  for (var test in invalidTests) {
    DebugLogger.warning("Warning: Test $test has no valid expectation.\n"
        "Expected one of: ${_multitestOutcomes.toString()}");

    outcomes.remove(test);
    testsAsLines.remove(test);
  }

  // Copy all the tests into the output map tests, as multiline strings.
  for (var key in testsAsLines.keys) {
    tests[key] = testsAsLines[key].join(lineSeparator);
  }
}

Future doMultitest(Path filePath, String outputDir, Path suiteDir,
    CreateTest doTest, bool hotReload) {
  void writeFile(String filepath, String content) {
    final File file = new File(filepath);

    if (file.existsSync()) {
      var oldContent = file.readAsStringSync();
      if (oldContent == content) {
        // Don't write to the file if the content is the same
        return;
      }
    }
    file.writeAsStringSync(content);
  }

  // Each new test is a single String value in the Map tests.
  var tests = <String, String>{};
  var outcomes = <String, Set<String>>{};
  extractTestsFromMultitest(filePath, tests, outcomes);

  var sourceDir = filePath.directoryPath;
  var targetDir = _createMultitestDirectory(outputDir, suiteDir, sourceDir);
  assert(targetDir != null);

  // Copy all the relative imports of the multitest.
  var importsToCopy = _findAllRelativeImports(filePath);
  var futureCopies = <Future>[];
  for (var relativeImport in importsToCopy) {
    var importPath = new Path(relativeImport);
    // Make sure the target directory exists.
    var importDir = importPath.directoryPath;
    if (!importDir.isEmpty) {
      TestUtils.mkdirRecursive(targetDir, importDir);
    }

    // Copy file.
    futureCopies.add(TestUtils.copyFile(
        sourceDir.join(importPath), targetDir.join(importPath)));
  }

  // Wait until all imports are copied before scheduling test cases.
  return Future.wait(futureCopies).then((_) {
    var baseFilename = filePath.filenameWithoutExtension;
    for (var key in tests.keys) {
      var multitestFilename = targetDir.append('${baseFilename}_$key.dart');
      writeFile(multitestFilename.toNativePath(), tests[key]);

      var outcome = outcomes[key];
      var hasStaticWarning = outcome.contains('static type warning');
      var hasRuntimeError = outcome.contains('runtime error');
      var hasSyntaxError = outcome.contains('syntax error');
      var hasCompileError =
          hasSyntaxError || outcome.contains('compile-time error');
      var isNegativeIfChecked = outcome.contains('dynamic type error');
      var hasCompileErrorIfChecked =
          outcome.contains('checked mode compile-time error');

      if (hotReload) {
        if (hasCompileError || hasCompileErrorIfChecked) {
          // Running a test that expects a compilation error with hot reloading
          // is redundant with a regular run of the test.
          continue;
        }
      }

      doTest(multitestFilename, filePath,
          hasSyntaxError: hasSyntaxError,
          hasCompileError: hasCompileError,
          hasRuntimeError: hasRuntimeError,
          isNegativeIfChecked: isNegativeIfChecked,
          hasCompileErrorIfChecked: hasCompileErrorIfChecked,
          hasStaticWarning: hasStaticWarning,
          multitestKey: key);
    }

    return null;
  });
}

/// A multitest annotation in the special `//#` comment.
class _Annotation {
  /// Parses the annotation in [line] or returns `null` if the line isn't a
  /// multitest annotation.
  static _Annotation tryParse(String line) {
    // Do an early return with "null" if this is not a valid multitest
    // annotation.
    if (!line.contains(_multitestMarker)) return null;

    var parts = line
        .split(_multitestMarker)[1]
        .split(':')
        .map((s) => s.trim())
        .where((s) => s.length > 0)
        .toList();

    if (parts.length <= 1) return null;

    return new _Annotation._(parts[0], parts[1]);
  }

  final String key;
  final String rest;

  // TODO(rnystrom): After Dart 1.0 is no longer supported, I don't think we
  // need to support more than a single outcome for each test.
  final List<String> outcomes = [];

  _Annotation._(this.key, this.rest) {
    outcomes.addAll(rest.split(',').map((s) => s.trim()));
  }
}

/// Finds all relative imports and copies them into the directory with the
/// generated tests.
Set<String> _findAllRelativeImports(Path topLibrary) {
  var found = new Set<String>();
  var libraryDir = topLibrary.directoryPath;
  var relativeImportRegExp = new RegExp(
      '^(?:@.*\\s+)?' // Allow for a meta-data annotation.
      '(import|part)'
      '\\s+["\']'
      '(?!(dart:|dart-ext:|data:|package:|/))' // Look-ahead: not in package.
      '([^"\']*)' // The path to the imported file.
      '["\']');

  processFile(Path filePath) {
    var file = new File(filePath.toNativePath());
    for (var line in file.readAsLinesSync()) {
      var match = relativeImportRegExp.firstMatch(line);
      if (match == null) continue;
      var relativePath = match.group(3);

      // If a multitest deliberately imports a non-existent file, don't try to
      // include it.
      if (relativePath.contains("nonexistent")) continue;

      // Handle import cycles.
      if (!found.add(relativePath)) continue;

      if (relativePath.contains("..")) {
        // This is just for safety reasons, we don't want to unintentionally
        // clobber files relative to the destination dir when copying them
        // over.
        print("Relative import in multitest containing '..' is not allowed.");
        exit(1);
      }

      processFile(libraryDir.append(relativePath));
    }
  }

  processFile(topLibrary);

  return found;
}

String _suiteNameFromPath(Path suiteDir) {
  var split = suiteDir.segments();

  // co19 test suite is at tests/co19/src.
  if (split.last == 'src') split.removeLast();

  return split.last;
}

Path _createMultitestDirectory(
    String outputDir, Path suiteDir, Path sourceDir) {
  var relative = sourceDir.relativeTo(suiteDir);
  var path = new Path(outputDir)
      .append('generated_tests')
      .append(_suiteNameFromPath(suiteDir))
      .join(relative);
  TestUtils.mkdirRecursive(Path.workingDirectory, path);
  return new Path(new File(path.toNativePath()).absolute.path);
}