| // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import "dart:async" show Future, Stream; |
| |
| import "dart:convert" show utf8; |
| |
| import "dart:io" show File; |
| |
| import "dart:typed_data" show Uint8List; |
| |
| import 'package:_fe_analyzer_shared/src/messages/diagnostic_message.dart' |
| show DiagnosticMessage, getMessageCodeObject; |
| |
| import 'package:_fe_analyzer_shared/src/messages/severity.dart' |
| show Severity, severityEnumValues; |
| |
| import 'package:kernel/ast.dart' show Location, Source; |
| |
| import "package:kernel/target/targets.dart" show TargetFlags; |
| |
| import "package:testing/testing.dart" |
| show Chain, ChainContext, Expectation, Result, Step, TestDescription, runMe; |
| |
| import "package:vm/target/vm.dart" show VmTarget; |
| |
| import "package:yaml/yaml.dart" show YamlList, YamlMap, YamlNode, loadYamlNode; |
| |
| import 'package:front_end/src/api_prototype/compiler_options.dart' |
| show CompilerOptions; |
| |
| import 'package:front_end/src/api_prototype/experimental_flags.dart' |
| show ExperimentalFlag; |
| |
| import 'package:front_end/src/api_prototype/memory_file_system.dart' |
| show MemoryFileSystem; |
| |
| import 'package:front_end/src/base/nnbd_mode.dart' show NnbdMode; |
| |
| import 'package:front_end/src/compute_platform_binaries_location.dart' |
| show computePlatformBinariesLocation; |
| |
| import 'package:front_end/src/fasta/command_line_reporting.dart' |
| as command_line_reporting; |
| |
| import 'package:front_end/src/fasta/hybrid_file_system.dart' |
| show HybridFileSystem; |
| |
| import "../../tool/_fasta/entry_points.dart" show BatchCompiler; |
| |
| import '../spell_checking_utils.dart' as spell; |
| |
| class MessageTestDescription extends TestDescription { |
| @override |
| final Uri uri; |
| |
| @override |
| final String shortName; |
| |
| final String name; |
| |
| final YamlMap data; |
| |
| final Example example; |
| |
| final String problem; |
| |
| MessageTestDescription(this.uri, this.shortName, this.name, this.data, |
| this.example, this.problem); |
| } |
| |
| class Configuration { |
| final NnbdMode nnbdMode; |
| |
| const Configuration(this.nnbdMode); |
| |
| CompilerOptions apply(CompilerOptions options) { |
| if (nnbdMode != null) { |
| options.experimentalFlags[ExperimentalFlag.nonNullable] = true; |
| options.nnbdMode = nnbdMode; |
| } |
| return options; |
| } |
| |
| static const Configuration defaultConfiguration = const Configuration(null); |
| } |
| |
| class MessageTestSuite extends ChainContext { |
| final List<Step> steps = const <Step>[ |
| const Validate(), |
| const Compile(), |
| ]; |
| |
| final MemoryFileSystem fileSystem; |
| |
| final BatchCompiler compiler; |
| |
| final bool fastOnly; |
| |
| MessageTestSuite(this.fastOnly) |
| : fileSystem = new MemoryFileSystem(Uri.parse("org-dartlang-fasta:///")), |
| compiler = new BatchCompiler(null); |
| |
| @override |
| Set<Expectation> processExpectedOutcomes( |
| Set<Expectation> outcomes, TestDescription description) { |
| if (description.shortName.contains("/spelling")) { |
| return {Expectation.Pass}; |
| } |
| return outcomes; |
| } |
| |
| /// Convert all the examples found in `messages.yaml` to a test |
| /// description. In addition, create a test description for each kind of |
| /// problem that a message can have. This problem will then be reported as a |
| /// failure by the [Validate] step that can be suppressed via the status |
| /// file. |
| Stream<MessageTestDescription> list(Chain suite) async* { |
| Uri uri = suite.uri.resolve("messages.yaml"); |
| File file = new File.fromUri(uri); |
| String fileContent = file.readAsStringSync(); |
| YamlMap messages = loadYamlNode(fileContent, sourceUrl: uri); |
| for (String name in messages.keys) { |
| YamlNode messageNode = messages.nodes[name]; |
| var message = messageNode.value; |
| if (message is String) continue; |
| |
| List<String> unknownKeys = <String>[]; |
| bool exampleAllowMoreCodes = false; |
| List<Example> examples = <Example>[]; |
| String externalTest; |
| bool frontendInternal = false; |
| List<String> analyzerCodes; |
| Severity severity; |
| YamlNode badSeverity; |
| YamlNode unnecessarySeverity; |
| List<String> badHasPublishedDocsValue = <String>[]; |
| List<String> spellingMessages; |
| const String spellingPostMessage = "\nIf the word(s) look okay, update " |
| "'spell_checking_list_messages.txt' or " |
| "'spell_checking_list_common.txt'."; |
| Configuration configuration; |
| |
| Source source; |
| List<String> formatSpellingMistakes( |
| spell.SpellingResult spellResult, int offset, String message) { |
| if (source == null) { |
| List<int> bytes = file.readAsBytesSync(); |
| List<int> lineStarts = new List<int>(); |
| int indexOf = 0; |
| while (indexOf >= 0) { |
| lineStarts.add(indexOf); |
| indexOf = bytes.indexOf(10, indexOf + 1); |
| } |
| lineStarts.add(bytes.length); |
| source = new Source(lineStarts, bytes, uri, uri); |
| } |
| List<String> result = new List<String>(); |
| for (int i = 0; i < spellResult.misspelledWords.length; i++) { |
| Location location = source.getLocation( |
| uri, offset + spellResult.misspelledWordsOffset[i]); |
| result.add(command_line_reporting.formatErrorMessage( |
| source.getTextLine(location.line), |
| location, |
| spellResult.misspelledWords[i].length, |
| relativize(uri), |
| "$message: '${spellResult.misspelledWords[i]}'.")); |
| } |
| return result; |
| } |
| |
| for (String key in message.keys) { |
| YamlNode node = message.nodes[key]; |
| var value = node.value; |
| // When positions matter, use node.span.text. |
| // When using node.span.text, replace r"\n" with "\n\n" to replace two |
| // characters with two characters without actually having the string |
| // "backslash n". |
| switch (key) { |
| case "template": |
| spell.SpellingResult spellingResult = spell.spellcheckString( |
| node.span.text.replaceAll(r"\n", "\n\n"), |
| dictionaries: const [ |
| spell.Dictionaries.common, |
| spell.Dictionaries.cfeMessages |
| ]); |
| if (spellingResult.misspelledWords != null) { |
| spellingMessages ??= new List<String>(); |
| spellingMessages.addAll(formatSpellingMistakes( |
| spellingResult, |
| node.span.start.offset, |
| "Template likely has the following spelling mistake")); |
| } |
| break; |
| |
| case "tip": |
| spell.SpellingResult spellingResult = spell.spellcheckString( |
| node.span.text.replaceAll(r"\n", "\n\n"), |
| dictionaries: const [ |
| spell.Dictionaries.common, |
| spell.Dictionaries.cfeMessages |
| ]); |
| if (spellingResult.misspelledWords != null) { |
| spellingMessages ??= new List<String>(); |
| spellingMessages.addAll(formatSpellingMistakes( |
| spellingResult, |
| node.span.start.offset, |
| "Tip likely has the following spelling mistake")); |
| } |
| break; |
| |
| case "severity": |
| severity = severityEnumValues[value]; |
| if (severity == null) { |
| badSeverity = node; |
| } else if (severity == Severity.error) { |
| unnecessarySeverity = node; |
| } |
| break; |
| |
| case "frontendInternal": |
| frontendInternal = value; |
| break; |
| |
| case "analyzerCode": |
| analyzerCodes = value is String |
| ? <String>[value] |
| : new List<String>.from(value); |
| break; |
| |
| case "exampleAllowMoreCodes": |
| exampleAllowMoreCodes = value; |
| break; |
| |
| case "bytes": |
| YamlList list = node; |
| if (list.first is List) { |
| for (YamlList bytes in list.nodes) { |
| int i = 0; |
| examples.add(new BytesExample("bytes${++i}", name, bytes)); |
| } |
| } else { |
| examples.add(new BytesExample("bytes", name, list)); |
| } |
| break; |
| |
| case "declaration": |
| if (node is YamlList) { |
| int i = 0; |
| for (YamlNode declaration in node.nodes) { |
| examples.add(new DeclarationExample( |
| "declaration${++i}", name, declaration)); |
| } |
| } else { |
| examples.add(new DeclarationExample("declaration", name, node)); |
| } |
| break; |
| |
| case "expression": |
| if (node is YamlList) { |
| int i = 0; |
| for (YamlNode expression in node.nodes) { |
| examples.add(new ExpressionExample( |
| "expression${++i}", name, expression)); |
| } |
| } else { |
| examples.add(new ExpressionExample("expression", name, node)); |
| } |
| break; |
| |
| case "script": |
| if (node is YamlList) { |
| int i = 0; |
| for (YamlNode script in node.nodes) { |
| examples |
| .add(new ScriptExample("script${++i}", name, script, this)); |
| } |
| } else { |
| examples.add(new ScriptExample("script", name, node, this)); |
| } |
| break; |
| |
| case "statement": |
| if (node is YamlList) { |
| int i = 0; |
| for (YamlNode statement in node.nodes) { |
| examples.add( |
| new StatementExample("statement${++i}", name, statement)); |
| } |
| } else { |
| examples.add(new StatementExample("statement", name, node)); |
| } |
| break; |
| |
| case "external": |
| externalTest = node.value; |
| break; |
| |
| case "index": |
| // index is validated during generation |
| break; |
| |
| case "hasPublishedDocs": |
| if (value != true) { |
| badHasPublishedDocsValue.add(name); |
| } |
| break; |
| |
| case "configuration": |
| if (value == "nnbd-weak") { |
| configuration = const Configuration(NnbdMode.Weak); |
| } else if (value == "nnbd-strong") { |
| configuration = const Configuration(NnbdMode.Strong); |
| } else { |
| throw new ArgumentError("Unknown configuration '$value'."); |
| } |
| break; |
| |
| default: |
| unknownKeys.add(key); |
| } |
| } |
| |
| if (exampleAllowMoreCodes) { |
| // Update all examples. |
| for (Example example in examples) { |
| example.allowMoreCodes = exampleAllowMoreCodes; |
| } |
| } |
| for (Example example in examples) { |
| example.configuration = |
| configuration ?? Configuration.defaultConfiguration; |
| } |
| |
| MessageTestDescription createDescription( |
| String subName, Example example, String problem, |
| {location}) { |
| String shortName = "$name/$subName"; |
| if (problem != null) { |
| String filename = relativize(uri); |
| location ??= message.span.start; |
| int line = location.line + 1; |
| int column = location.column; |
| problem = "$filename:$line:$column: error:\n$problem"; |
| } |
| return new MessageTestDescription(uri.resolve("#$shortName"), shortName, |
| name, messageNode, example, problem); |
| } |
| |
| if (!fastOnly) { |
| for (Example example in examples) { |
| yield createDescription(example.name, example, null); |
| } |
| // "Wrap" example as a part. |
| for (Example example in examples) { |
| yield createDescription( |
| "part_wrapped_${example.name}", |
| new PartWrapExample("part_wrapped_${example.name}", name, |
| exampleAllowMoreCodes, example), |
| null); |
| } |
| } |
| |
| yield createDescription( |
| "knownKeys", |
| null, |
| unknownKeys.isNotEmpty |
| ? "Unknown keys: ${unknownKeys.join(' ')}." |
| : null); |
| |
| yield createDescription( |
| 'hasPublishedDocs', |
| null, |
| badHasPublishedDocsValue.isNotEmpty |
| ? "Bad hasPublishedDocs value (only 'true' supported) in:" |
| " ${badHasPublishedDocsValue.join(', ')}" |
| : null); |
| |
| yield createDescription( |
| "severity", |
| null, |
| badSeverity != null |
| ? "Unknown severity: '${badSeverity.value}'." |
| : null, |
| location: badSeverity?.span?.start); |
| |
| yield createDescription( |
| "unnecessarySeverity", |
| null, |
| unnecessarySeverity != null |
| ? "The 'ERROR' severity is the default and not necessary." |
| : null, |
| location: unnecessarySeverity?.span?.start); |
| |
| yield createDescription( |
| "spelling", |
| null, |
| spellingMessages != null |
| ? spellingMessages.join("\n") + spellingPostMessage |
| : null); |
| |
| bool exampleAndAnalyzerCodeRequired = severity != Severity.context && |
| severity != Severity.internalProblem && |
| severity != Severity.ignored; |
| |
| yield createDescription( |
| "externalExample", |
| null, |
| exampleAndAnalyzerCodeRequired && |
| externalTest != null && |
| !(new File.fromUri(suite.uri.resolve(externalTest)) |
| .existsSync()) |
| ? "Given external example for $name points to a nonexisting file " |
| "(${suite.uri.resolve(externalTest)})." |
| : null); |
| |
| yield createDescription( |
| "example", |
| null, |
| exampleAndAnalyzerCodeRequired && |
| examples.isEmpty && |
| externalTest == null |
| ? "No example for $name, please add at least one example." |
| : null); |
| |
| yield createDescription( |
| "analyzerCode", |
| null, |
| exampleAndAnalyzerCodeRequired && |
| !frontendInternal && |
| analyzerCodes == null |
| ? "No analyzer code for $name." |
| "\nTry running" |
| " <BUILDDIR>/dart-sdk/bin/dartanalyzer --format=machine" |
| " on an example to find the code." |
| " The code is printed just before the file name." |
| : null); |
| } |
| } |
| |
| String formatProblems( |
| String message, Example example, List<DiagnosticMessage> messages) { |
| var span = example.node.span; |
| StringBuffer buffer = new StringBuffer(); |
| buffer |
| ..write(relativize(span.sourceUrl)) |
| ..write(":") |
| ..write(span.start.line + 1) |
| ..write(":") |
| ..write(span.start.column) |
| ..write(": error: ") |
| ..write(message); |
| buffer.write("\n${span.text}"); |
| for (DiagnosticMessage message in messages) { |
| buffer.write("\nCode: ${getMessageCodeObject(message).name}"); |
| buffer.write("\n > "); |
| buffer.write( |
| message.plainTextFormatted.join("\n").replaceAll("\n", "\n > ")); |
| } |
| |
| return "$buffer"; |
| } |
| } |
| |
| abstract class Example { |
| final String name; |
| |
| final String expectedCode; |
| |
| bool allowMoreCodes = false; |
| |
| Configuration configuration; |
| |
| Example(this.name, this.expectedCode); |
| |
| YamlNode get node; |
| |
| Uint8List get bytes; |
| |
| Map<String, Uint8List> get scripts { |
| return {mainFilename: bytes}; |
| } |
| |
| String get mainFilename => "main.dart"; |
| } |
| |
| class BytesExample extends Example { |
| @override |
| final YamlList node; |
| |
| @override |
| final Uint8List bytes; |
| |
| BytesExample(String name, String code, this.node) |
| : bytes = new Uint8List.fromList(node.cast<int>()), |
| super(name, code); |
| } |
| |
| class DeclarationExample extends Example { |
| @override |
| final YamlNode node; |
| |
| final String declaration; |
| |
| DeclarationExample(String name, String code, this.node) |
| : declaration = node.value, |
| super(name, code); |
| |
| @override |
| Uint8List get bytes { |
| return new Uint8List.fromList(utf8.encode(""" |
| $declaration |
| |
| main() { |
| } |
| """)); |
| } |
| } |
| |
| class StatementExample extends Example { |
| @override |
| final YamlNode node; |
| |
| final String statement; |
| |
| StatementExample(String name, String code, this.node) |
| : statement = node.value, |
| super(name, code); |
| |
| @override |
| Uint8List get bytes { |
| return new Uint8List.fromList(utf8.encode(""" |
| main() { |
| $statement |
| } |
| """)); |
| } |
| } |
| |
| class ExpressionExample extends Example { |
| @override |
| final YamlNode node; |
| |
| final String expression; |
| |
| ExpressionExample(String name, String code, this.node) |
| : expression = node.value, |
| super(name, code); |
| |
| @override |
| Uint8List get bytes { |
| return new Uint8List.fromList(utf8.encode(""" |
| main() { |
| $expression; |
| } |
| """)); |
| } |
| } |
| |
| class ScriptExample extends Example { |
| @override |
| final YamlNode node; |
| |
| final Object script; |
| |
| ScriptExample(String name, String code, this.node, MessageTestSuite suite) |
| : script = node.value, |
| super(name, code) { |
| if (script is! String && script is! Map) { |
| throw suite.formatProblems( |
| "A script must be either a String or a Map in $code:", |
| this, <DiagnosticMessage>[]); |
| } |
| } |
| |
| @override |
| Uint8List get bytes => throw "Unsupported: ScriptExample.bytes"; |
| |
| @override |
| Map<String, Uint8List> get scripts { |
| Object script = this.script; |
| if (script is Map) { |
| var scriptFiles = <String, Uint8List>{}; |
| script.forEach((fileName, value) { |
| scriptFiles[fileName] = new Uint8List.fromList(utf8.encode(value)); |
| print("$fileName => $value\n\n======\n\n"); |
| }); |
| return scriptFiles; |
| } else { |
| return {mainFilename: new Uint8List.fromList(utf8.encode(script))}; |
| } |
| } |
| } |
| |
| class PartWrapExample extends Example { |
| final Example example; |
| @override |
| final bool allowMoreCodes; |
| |
| PartWrapExample(String name, String code, this.allowMoreCodes, this.example) |
| : super(name, code) { |
| configuration = example.configuration; |
| } |
| |
| @override |
| Uint8List get bytes => throw "Unsupported: PartWrapExample.bytes"; |
| |
| @override |
| String get mainFilename => "main_wrapped.dart"; |
| |
| @override |
| Map<String, Uint8List> get scripts { |
| Map<String, Uint8List> wrapped = example.scripts; |
| |
| var scriptFiles = <String, Uint8List>{}; |
| scriptFiles.addAll(wrapped); |
| |
| // Create a new main file |
| // TODO: Technically we should find a un-used name. |
| if (scriptFiles.containsKey(mainFilename)) { |
| throw "Framework failure: " |
| "Wanted to create wrapper file, but the file already exists!"; |
| } |
| scriptFiles[mainFilename] = new Uint8List.fromList(utf8.encode(""" |
| part "${example.mainFilename}"; |
| """)); |
| |
| // Modify the original main file to be part of the wrapper and add lots of |
| // gunk so every actual position in the file is not a valid position in the |
| // wrapper. |
| scriptFiles[example.mainFilename] = new Uint8List.fromList(utf8.encode(""" |
| part of "${mainFilename}"; |
| // La la la la la la la la la la la la la. |
| // La la la la la la la la la la la la la. |
| // La la la la la la la la la la la la la. |
| // La la la la la la la la la la la la la. |
| // La la la la la la la la la la la la la. |
| """) + scriptFiles[example.mainFilename]); |
| |
| return scriptFiles; |
| } |
| |
| @override |
| YamlNode get node => example.node; |
| } |
| |
| class Validate extends Step<MessageTestDescription, Example, MessageTestSuite> { |
| const Validate(); |
| |
| String get name => "validate"; |
| |
| Future<Result<Example>> run( |
| MessageTestDescription description, MessageTestSuite suite) async { |
| if (description.problem != null) { |
| return fail(null, description.problem); |
| } else { |
| return pass(description.example); |
| } |
| } |
| } |
| |
| class Compile extends Step<Example, Null, MessageTestSuite> { |
| const Compile(); |
| |
| String get name => "compile"; |
| |
| Future<Result<Null>> run(Example example, MessageTestSuite suite) async { |
| if (example == null) return pass(null); |
| String dir = "${example.expectedCode}/${example.name}"; |
| example.scripts.forEach((String fileName, Uint8List bytes) { |
| Uri uri = suite.fileSystem.currentDirectory.resolve("$dir/$fileName"); |
| suite.fileSystem.entityForUri(uri).writeAsBytesSync(bytes); |
| }); |
| Uri main = suite.fileSystem.currentDirectory |
| .resolve("$dir/${example.mainFilename}"); |
| Uri output = |
| suite.fileSystem.currentDirectory.resolve("$dir/main.dart.dill"); |
| |
| // Setup .packages if it (or package_config.json) doesn't exist. |
| Uri dotPackagesUri = |
| suite.fileSystem.currentDirectory.resolve("$dir/.packages"); |
| Uri packageConfigJsonUri = suite.fileSystem.currentDirectory |
| .resolve("$dir/.dart_tool/package_config.json"); |
| if (!await suite.fileSystem.entityForUri(dotPackagesUri).exists() && |
| !await suite.fileSystem.entityForUri(packageConfigJsonUri).exists()) { |
| suite.fileSystem.entityForUri(dotPackagesUri).writeAsBytesSync([]); |
| } |
| |
| print("Compiling $main"); |
| List<DiagnosticMessage> messages = <DiagnosticMessage>[]; |
| |
| await suite.compiler.batchCompile( |
| example.configuration.apply(new CompilerOptions() |
| ..sdkSummary = computePlatformBinariesLocation(forceBuildDir: true) |
| .resolve("vm_platform_strong.dill") |
| ..target = new VmTarget(new TargetFlags()) |
| ..fileSystem = new HybridFileSystem(suite.fileSystem) |
| ..packagesFileUri = dotPackagesUri |
| ..onDiagnostic = messages.add |
| ..environmentDefines = const {}), |
| main, |
| output); |
| |
| List<DiagnosticMessage> unexpectedMessages = <DiagnosticMessage>[]; |
| if (example.allowMoreCodes) { |
| List<DiagnosticMessage> messagesFiltered = new List<DiagnosticMessage>(); |
| for (DiagnosticMessage message in messages) { |
| if (getMessageCodeObject(message).name == example.expectedCode) { |
| messagesFiltered.add(message); |
| } |
| } |
| messages = messagesFiltered; |
| } |
| for (DiagnosticMessage message in messages) { |
| if (getMessageCodeObject(message).name != example.expectedCode) { |
| unexpectedMessages.add(message); |
| } |
| } |
| if (unexpectedMessages.isEmpty) { |
| switch (messages.length) { |
| case 0: |
| return fail( |
| null, |
| suite.formatProblems("No message reported in ${example.name}:", |
| example, messages)); |
| case 1: |
| return pass(null); |
| default: |
| return fail( |
| null, |
| suite.formatProblems( |
| "Message reported multiple times in ${example.name}:", |
| example, |
| messages)); |
| } |
| } |
| return fail( |
| null, |
| suite.formatProblems("Too many messages reported in ${example.name}:", |
| example, messages)); |
| } |
| } |
| |
| Future<MessageTestSuite> createContext( |
| Chain suite, Map<String, String> environment) async { |
| final bool fastOnly = environment["fastOnly"] == "true"; |
| return new MessageTestSuite(fastOnly); |
| } |
| |
| String relativize(Uri uri) { |
| String base = "${Uri.base}"; |
| String filename = "$uri"; |
| if (filename.startsWith(base)) { |
| return filename.substring(base.length); |
| } else { |
| return filename; |
| } |
| } |
| |
| main([List<String> arguments = const []]) => |
| runMe(arguments, createContext, configurationPath: "../../testing.json"); |
