blob: cb0f930e7161fd8b9ba1e588ef5b3b9b73e1c115 [file] [log] [blame]
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import "dart:async" show Future, Stream;
import "dart:convert" show utf8;
import "dart:io" show File;
import "dart:typed_data" show Uint8List;
import 'package:_fe_analyzer_shared/src/messages/diagnostic_message.dart'
show DiagnosticMessage, getMessageCodeObject;
import 'package:_fe_analyzer_shared/src/messages/severity.dart'
show Severity, severityEnumValues;
import 'package:kernel/ast.dart' show Location, Source;
import "package:kernel/target/targets.dart" show TargetFlags;
import "package:testing/testing.dart"
show Chain, ChainContext, Expectation, Result, Step, TestDescription, runMe;
import "package:vm/target/vm.dart" show VmTarget;
import "package:yaml/yaml.dart" show YamlList, YamlMap, YamlNode, loadYamlNode;
import 'package:front_end/src/api_prototype/compiler_options.dart'
show CompilerOptions;
import 'package:front_end/src/api_prototype/experimental_flags.dart'
show ExperimentalFlag;
import 'package:front_end/src/api_prototype/memory_file_system.dart'
show MemoryFileSystem;
import 'package:front_end/src/base/nnbd_mode.dart' show NnbdMode;
import 'package:front_end/src/compute_platform_binaries_location.dart'
show computePlatformBinariesLocation;
import 'package:front_end/src/fasta/command_line_reporting.dart'
as command_line_reporting;
import 'package:front_end/src/fasta/hybrid_file_system.dart'
show HybridFileSystem;
import "../../tool/_fasta/entry_points.dart" show BatchCompiler;
import '../spell_checking_utils.dart' as spell;
class MessageTestDescription extends TestDescription {
@override
final Uri uri;
@override
final String shortName;
final String name;
final YamlMap data;
final Example example;
final String problem;
MessageTestDescription(this.uri, this.shortName, this.name, this.data,
this.example, this.problem);
}
class Configuration {
final NnbdMode nnbdMode;
const Configuration(this.nnbdMode);
CompilerOptions apply(CompilerOptions options) {
if (nnbdMode != null) {
options.experimentalFlags[ExperimentalFlag.nonNullable] = true;
options.nnbdMode = nnbdMode;
}
return options;
}
static const Configuration defaultConfiguration = const Configuration(null);
}
class MessageTestSuite extends ChainContext {
final List<Step> steps = const <Step>[
const Validate(),
const Compile(),
];
final MemoryFileSystem fileSystem;
final BatchCompiler compiler;
final bool fastOnly;
MessageTestSuite(this.fastOnly)
: fileSystem = new MemoryFileSystem(Uri.parse("org-dartlang-fasta:///")),
compiler = new BatchCompiler(null);
@override
Set<Expectation> processExpectedOutcomes(
Set<Expectation> outcomes, TestDescription description) {
if (description.shortName.contains("/spelling")) {
return {Expectation.Pass};
}
return outcomes;
}
/// Convert all the examples found in `messages.yaml` to a test
/// description. In addition, create a test description for each kind of
/// problem that a message can have. This problem will then be reported as a
/// failure by the [Validate] step that can be suppressed via the status
/// file.
Stream<MessageTestDescription> list(Chain suite) async* {
Uri uri = suite.uri.resolve("messages.yaml");
File file = new File.fromUri(uri);
String fileContent = file.readAsStringSync();
YamlMap messages = loadYamlNode(fileContent, sourceUrl: uri);
for (String name in messages.keys) {
YamlNode messageNode = messages.nodes[name];
var message = messageNode.value;
if (message is String) continue;
List<String> unknownKeys = <String>[];
bool exampleAllowMoreCodes = false;
List<Example> examples = <Example>[];
String externalTest;
bool frontendInternal = false;
List<String> analyzerCodes;
Severity severity;
YamlNode badSeverity;
YamlNode unnecessarySeverity;
List<String> badHasPublishedDocsValue = <String>[];
List<String> spellingMessages;
const String spellingPostMessage = "\nIf the word(s) look okay, update "
"'spell_checking_list_messages.txt' or "
"'spell_checking_list_common.txt'.";
Configuration configuration;
Source source;
List<String> formatSpellingMistakes(
spell.SpellingResult spellResult, int offset, String message) {
if (source == null) {
List<int> bytes = file.readAsBytesSync();
List<int> lineStarts = new List<int>();
int indexOf = 0;
while (indexOf >= 0) {
lineStarts.add(indexOf);
indexOf = bytes.indexOf(10, indexOf + 1);
}
lineStarts.add(bytes.length);
source = new Source(lineStarts, bytes, uri, uri);
}
List<String> result = new List<String>();
for (int i = 0; i < spellResult.misspelledWords.length; i++) {
Location location = source.getLocation(
uri, offset + spellResult.misspelledWordsOffset[i]);
result.add(command_line_reporting.formatErrorMessage(
source.getTextLine(location.line),
location,
spellResult.misspelledWords[i].length,
relativize(uri),
"$message: '${spellResult.misspelledWords[i]}'."));
}
return result;
}
for (String key in message.keys) {
YamlNode node = message.nodes[key];
var value = node.value;
// When positions matter, use node.span.text.
// When using node.span.text, replace r"\n" with "\n\n" to replace two
// characters with two characters without actually having the string
// "backslash n".
switch (key) {
case "template":
spell.SpellingResult spellingResult = spell.spellcheckString(
node.span.text.replaceAll(r"\n", "\n\n"),
dictionaries: const [
spell.Dictionaries.common,
spell.Dictionaries.cfeMessages
]);
if (spellingResult.misspelledWords != null) {
spellingMessages ??= new List<String>();
spellingMessages.addAll(formatSpellingMistakes(
spellingResult,
node.span.start.offset,
"Template likely has the following spelling mistake"));
}
break;
case "tip":
spell.SpellingResult spellingResult = spell.spellcheckString(
node.span.text.replaceAll(r"\n", "\n\n"),
dictionaries: const [
spell.Dictionaries.common,
spell.Dictionaries.cfeMessages
]);
if (spellingResult.misspelledWords != null) {
spellingMessages ??= new List<String>();
spellingMessages.addAll(formatSpellingMistakes(
spellingResult,
node.span.start.offset,
"Tip likely has the following spelling mistake"));
}
break;
case "severity":
severity = severityEnumValues[value];
if (severity == null) {
badSeverity = node;
} else if (severity == Severity.error) {
unnecessarySeverity = node;
}
break;
case "frontendInternal":
frontendInternal = value;
break;
case "analyzerCode":
analyzerCodes = value is String
? <String>[value]
: new List<String>.from(value);
break;
case "exampleAllowMoreCodes":
exampleAllowMoreCodes = value;
break;
case "bytes":
YamlList list = node;
if (list.first is List) {
for (YamlList bytes in list.nodes) {
int i = 0;
examples.add(new BytesExample("bytes${++i}", name, bytes));
}
} else {
examples.add(new BytesExample("bytes", name, list));
}
break;
case "declaration":
if (node is YamlList) {
int i = 0;
for (YamlNode declaration in node.nodes) {
examples.add(new DeclarationExample(
"declaration${++i}", name, declaration));
}
} else {
examples.add(new DeclarationExample("declaration", name, node));
}
break;
case "expression":
if (node is YamlList) {
int i = 0;
for (YamlNode expression in node.nodes) {
examples.add(new ExpressionExample(
"expression${++i}", name, expression));
}
} else {
examples.add(new ExpressionExample("expression", name, node));
}
break;
case "script":
if (node is YamlList) {
int i = 0;
for (YamlNode script in node.nodes) {
examples
.add(new ScriptExample("script${++i}", name, script, this));
}
} else {
examples.add(new ScriptExample("script", name, node, this));
}
break;
case "statement":
if (node is YamlList) {
int i = 0;
for (YamlNode statement in node.nodes) {
examples.add(
new StatementExample("statement${++i}", name, statement));
}
} else {
examples.add(new StatementExample("statement", name, node));
}
break;
case "external":
externalTest = node.value;
break;
case "index":
// index is validated during generation
break;
case "hasPublishedDocs":
if (value != true) {
badHasPublishedDocsValue.add(name);
}
break;
case "configuration":
if (value == "nnbd-weak") {
configuration = const Configuration(NnbdMode.Weak);
} else if (value == "nnbd-strong") {
configuration = const Configuration(NnbdMode.Strong);
} else {
throw new ArgumentError("Unknown configuration '$value'.");
}
break;
default:
unknownKeys.add(key);
}
}
if (exampleAllowMoreCodes) {
// Update all examples.
for (Example example in examples) {
example.allowMoreCodes = exampleAllowMoreCodes;
}
}
for (Example example in examples) {
example.configuration =
configuration ?? Configuration.defaultConfiguration;
}
MessageTestDescription createDescription(
String subName, Example example, String problem,
{location}) {
String shortName = "$name/$subName";
if (problem != null) {
String filename = relativize(uri);
location ??= message.span.start;
int line = location.line + 1;
int column = location.column;
problem = "$filename:$line:$column: error:\n$problem";
}
return new MessageTestDescription(uri.resolve("#$shortName"), shortName,
name, messageNode, example, problem);
}
if (!fastOnly) {
for (Example example in examples) {
yield createDescription(example.name, example, null);
}
// "Wrap" example as a part.
for (Example example in examples) {
yield createDescription(
"part_wrapped_${example.name}",
new PartWrapExample("part_wrapped_${example.name}", name,
exampleAllowMoreCodes, example),
null);
}
}
yield createDescription(
"knownKeys",
null,
unknownKeys.isNotEmpty
? "Unknown keys: ${unknownKeys.join(' ')}."
: null);
yield createDescription(
'hasPublishedDocs',
null,
badHasPublishedDocsValue.isNotEmpty
? "Bad hasPublishedDocs value (only 'true' supported) in:"
" ${badHasPublishedDocsValue.join(', ')}"
: null);
yield createDescription(
"severity",
null,
badSeverity != null
? "Unknown severity: '${badSeverity.value}'."
: null,
location: badSeverity?.span?.start);
yield createDescription(
"unnecessarySeverity",
null,
unnecessarySeverity != null
? "The 'ERROR' severity is the default and not necessary."
: null,
location: unnecessarySeverity?.span?.start);
yield createDescription(
"spelling",
null,
spellingMessages != null
? spellingMessages.join("\n") + spellingPostMessage
: null);
bool exampleAndAnalyzerCodeRequired = severity != Severity.context &&
severity != Severity.internalProblem &&
severity != Severity.ignored;
yield createDescription(
"externalExample",
null,
exampleAndAnalyzerCodeRequired &&
externalTest != null &&
!(new File.fromUri(suite.uri.resolve(externalTest))
.existsSync())
? "Given external example for $name points to a nonexisting file "
"(${suite.uri.resolve(externalTest)})."
: null);
yield createDescription(
"example",
null,
exampleAndAnalyzerCodeRequired &&
examples.isEmpty &&
externalTest == null
? "No example for $name, please add at least one example."
: null);
yield createDescription(
"analyzerCode",
null,
exampleAndAnalyzerCodeRequired &&
!frontendInternal &&
analyzerCodes == null
? "No analyzer code for $name."
"\nTry running"
" <BUILDDIR>/dart-sdk/bin/dartanalyzer --format=machine"
" on an example to find the code."
" The code is printed just before the file name."
: null);
}
}
String formatProblems(
String message, Example example, List<DiagnosticMessage> messages) {
var span = example.node.span;
StringBuffer buffer = new StringBuffer();
buffer
..write(relativize(span.sourceUrl))
..write(":")
..write(span.start.line + 1)
..write(":")
..write(span.start.column)
..write(": error: ")
..write(message);
buffer.write("\n${span.text}");
for (DiagnosticMessage message in messages) {
buffer.write("\nCode: ${getMessageCodeObject(message).name}");
buffer.write("\n > ");
buffer.write(
message.plainTextFormatted.join("\n").replaceAll("\n", "\n > "));
}
return "$buffer";
}
}
abstract class Example {
final String name;
final String expectedCode;
bool allowMoreCodes = false;
Configuration configuration;
Example(this.name, this.expectedCode);
YamlNode get node;
Uint8List get bytes;
Map<String, Uint8List> get scripts {
return {mainFilename: bytes};
}
String get mainFilename => "main.dart";
}
class BytesExample extends Example {
@override
final YamlList node;
@override
final Uint8List bytes;
BytesExample(String name, String code, this.node)
: bytes = new Uint8List.fromList(node.cast<int>()),
super(name, code);
}
class DeclarationExample extends Example {
@override
final YamlNode node;
final String declaration;
DeclarationExample(String name, String code, this.node)
: declaration = node.value,
super(name, code);
@override
Uint8List get bytes {
return new Uint8List.fromList(utf8.encode("""
$declaration
main() {
}
"""));
}
}
class StatementExample extends Example {
@override
final YamlNode node;
final String statement;
StatementExample(String name, String code, this.node)
: statement = node.value,
super(name, code);
@override
Uint8List get bytes {
return new Uint8List.fromList(utf8.encode("""
main() {
$statement
}
"""));
}
}
class ExpressionExample extends Example {
@override
final YamlNode node;
final String expression;
ExpressionExample(String name, String code, this.node)
: expression = node.value,
super(name, code);
@override
Uint8List get bytes {
return new Uint8List.fromList(utf8.encode("""
main() {
$expression;
}
"""));
}
}
class ScriptExample extends Example {
@override
final YamlNode node;
final Object script;
ScriptExample(String name, String code, this.node, MessageTestSuite suite)
: script = node.value,
super(name, code) {
if (script is! String && script is! Map) {
throw suite.formatProblems(
"A script must be either a String or a Map in $code:",
this, <DiagnosticMessage>[]);
}
}
@override
Uint8List get bytes => throw "Unsupported: ScriptExample.bytes";
@override
Map<String, Uint8List> get scripts {
Object script = this.script;
if (script is Map) {
var scriptFiles = <String, Uint8List>{};
script.forEach((fileName, value) {
scriptFiles[fileName] = new Uint8List.fromList(utf8.encode(value));
print("$fileName => $value\n\n======\n\n");
});
return scriptFiles;
} else {
return {mainFilename: new Uint8List.fromList(utf8.encode(script))};
}
}
}
class PartWrapExample extends Example {
final Example example;
@override
final bool allowMoreCodes;
PartWrapExample(String name, String code, this.allowMoreCodes, this.example)
: super(name, code) {
configuration = example.configuration;
}
@override
Uint8List get bytes => throw "Unsupported: PartWrapExample.bytes";
@override
String get mainFilename => "main_wrapped.dart";
@override
Map<String, Uint8List> get scripts {
Map<String, Uint8List> wrapped = example.scripts;
var scriptFiles = <String, Uint8List>{};
scriptFiles.addAll(wrapped);
// Create a new main file
// TODO: Technically we should find a un-used name.
if (scriptFiles.containsKey(mainFilename)) {
throw "Framework failure: "
"Wanted to create wrapper file, but the file already exists!";
}
scriptFiles[mainFilename] = new Uint8List.fromList(utf8.encode("""
part "${example.mainFilename}";
"""));
// Modify the original main file to be part of the wrapper and add lots of
// gunk so every actual position in the file is not a valid position in the
// wrapper.
scriptFiles[example.mainFilename] = new Uint8List.fromList(utf8.encode("""
part of "${mainFilename}";
// La la la la la la la la la la la la la.
// La la la la la la la la la la la la la.
// La la la la la la la la la la la la la.
// La la la la la la la la la la la la la.
// La la la la la la la la la la la la la.
""") + scriptFiles[example.mainFilename]);
return scriptFiles;
}
@override
YamlNode get node => example.node;
}
class Validate extends Step<MessageTestDescription, Example, MessageTestSuite> {
const Validate();
String get name => "validate";
Future<Result<Example>> run(
MessageTestDescription description, MessageTestSuite suite) async {
if (description.problem != null) {
return fail(null, description.problem);
} else {
return pass(description.example);
}
}
}
class Compile extends Step<Example, Null, MessageTestSuite> {
const Compile();
String get name => "compile";
Future<Result<Null>> run(Example example, MessageTestSuite suite) async {
if (example == null) return pass(null);
String dir = "${example.expectedCode}/${example.name}";
example.scripts.forEach((String fileName, Uint8List bytes) {
Uri uri = suite.fileSystem.currentDirectory.resolve("$dir/$fileName");
suite.fileSystem.entityForUri(uri).writeAsBytesSync(bytes);
});
Uri main = suite.fileSystem.currentDirectory
.resolve("$dir/${example.mainFilename}");
Uri output =
suite.fileSystem.currentDirectory.resolve("$dir/main.dart.dill");
// Setup .packages if it (or package_config.json) doesn't exist.
Uri dotPackagesUri =
suite.fileSystem.currentDirectory.resolve("$dir/.packages");
Uri packageConfigJsonUri = suite.fileSystem.currentDirectory
.resolve("$dir/.dart_tool/package_config.json");
if (!await suite.fileSystem.entityForUri(dotPackagesUri).exists() &&
!await suite.fileSystem.entityForUri(packageConfigJsonUri).exists()) {
suite.fileSystem.entityForUri(dotPackagesUri).writeAsBytesSync([]);
}
print("Compiling $main");
List<DiagnosticMessage> messages = <DiagnosticMessage>[];
await suite.compiler.batchCompile(
example.configuration.apply(new CompilerOptions()
..sdkSummary = computePlatformBinariesLocation(forceBuildDir: true)
.resolve("vm_platform_strong.dill")
..target = new VmTarget(new TargetFlags())
..fileSystem = new HybridFileSystem(suite.fileSystem)
..packagesFileUri = dotPackagesUri
..onDiagnostic = messages.add
..environmentDefines = const {}),
main,
output);
List<DiagnosticMessage> unexpectedMessages = <DiagnosticMessage>[];
if (example.allowMoreCodes) {
List<DiagnosticMessage> messagesFiltered = new List<DiagnosticMessage>();
for (DiagnosticMessage message in messages) {
if (getMessageCodeObject(message).name == example.expectedCode) {
messagesFiltered.add(message);
}
}
messages = messagesFiltered;
}
for (DiagnosticMessage message in messages) {
if (getMessageCodeObject(message).name != example.expectedCode) {
unexpectedMessages.add(message);
}
}
if (unexpectedMessages.isEmpty) {
switch (messages.length) {
case 0:
return fail(
null,
suite.formatProblems("No message reported in ${example.name}:",
example, messages));
case 1:
return pass(null);
default:
return fail(
null,
suite.formatProblems(
"Message reported multiple times in ${example.name}:",
example,
messages));
}
}
return fail(
null,
suite.formatProblems("Too many messages reported in ${example.name}:",
example, messages));
}
}
Future<MessageTestSuite> createContext(
Chain suite, Map<String, String> environment) async {
final bool fastOnly = environment["fastOnly"] == "true";
return new MessageTestSuite(fastOnly);
}
String relativize(Uri uri) {
String base = "${Uri.base}";
String filename = "$uri";
if (filename.startsWith(base)) {
return filename.substring(base.length);
} else {
return filename;
}
}
main([List<String> arguments = const []]) =>
runMe(arguments, createContext, configurationPath: "../../testing.json");