blob: f18f0d20e22044f6d695b464bc37e9ec4d3f3d2a [file] [log] [blame]
// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
/// This is an interface to the Dart Kernel parser and Kernel binary generator.
///
/// It is used by the kernel-isolate to load Dart source code and generate
/// Kernel binary format.
///
/// This is either invoked as the root script of the Kernel isolate when used
/// as a part of
///
/// dart --dfe=pkg/vm/bin/kernel_service.dart ...
///
/// invocation or it is invoked as a standalone script to perform training for
/// the app-jit snapshot
///
/// dart pkg/vm/bin/kernel_service.dart --train <source-file>
///
///
import 'dart:async' show Future, ZoneSpecification, runZoned;
import 'dart:collection' show UnmodifiableMapBase;
import 'dart:convert' show utf8;
import 'dart:io'
show Directory, File, Platform, stderr, stdout
hide FileSystemEntity;
import 'dart:isolate';
import 'dart:typed_data' show Uint8List;
import 'package:build_integration/file_system/multi_root.dart';
import 'package:front_end/src/api_prototype/front_end.dart' as fe
show CompilerResult;
import 'package:front_end/src/api_prototype/memory_file_system.dart';
import 'package:front_end/src/api_unstable/vm.dart';
import 'package:kernel/binary/ast_to_binary.dart';
import 'package:kernel/binary/ast_from_binary.dart'
show BinaryBuilderWithMetadata;
import 'package:kernel/class_hierarchy.dart' show ClassHierarchy;
import 'package:kernel/core_types.dart' show CoreTypes;
import 'package:kernel/kernel.dart' show Component, Library, Procedure;
import 'package:kernel/target/targets.dart' show TargetFlags;
import 'package:vm/incremental_compiler.dart';
import 'package:vm/kernel_front_end.dart'
show autoDetectNullSafetyMode, createLoadedLibrariesSet;
import 'package:vm/http_filesystem.dart';
import 'package:vm/target/vm.dart' show VmTarget;
final bool verbose = new bool.fromEnvironment('DFE_VERBOSE');
final bool dumpKernel = new bool.fromEnvironment('DFE_DUMP_KERNEL');
const String platformKernelFile = 'virtual_platform_kernel.dill';
const String dotPackagesFile = '.packages';
// NOTE: Any changes to these tags need to be reflected in kernel_isolate.cc
// Tags used to indicate different requests to the dart frontend.
//
// Current tags include the following:
// 0 - Perform normal compilation.
// 1 - Update in-memory file system with in-memory sources (used by tests).
// 2 - Accept last compilation result.
// 3 - APP JIT snapshot training run for kernel_service.
// 4 - Compile an individual expression in some context (for debugging
// purposes).
// 5 - List program dependencies (for creating depfiles)
// 6 - Isolate shutdown that potentially should result in compiler cleanup.
const int kCompileTag = 0;
const int kUpdateSourcesTag = 1;
const int kAcceptTag = 2;
const int kTrainTag = 3;
const int kCompileExpressionTag = 4;
const int kListDependenciesTag = 5;
const int kNotifyIsolateShutdownTag = 6;
const int kDetectNullabilityTag = 7;
bool allowDartInternalImport = false;
// Null Safety command line options
//
// Note: The values of these constants must match the
// values of flag sound_null_safety in ../../../../runtime/vm/flag_list.h.
// 0 - No --[no-]sound-null-safety option specified on the command line.
// 1 - '--no-sound-null-safety' specified on the command line.
// 2 - '--sound-null-safety' option specified on the command line.
const int kNullSafetyOptionUnspecified = 0;
const int kNullSafetyOptionWeak = 1;
const int kNullSafetyOptionStrong = 2;
CompilerOptions setupCompilerOptions(
FileSystem fileSystem,
Uri? platformKernelPath,
bool enableAsserts,
int nullSafety,
List<String>? experimentalFlags,
Uri? packagesUri,
List<String> errorsPlain,
List<String> errorsColorized,
String invocationModes,
String verbosityLevel) {
final expFlags = <String>[];
if (experimentalFlags != null) {
for (String flag in experimentalFlags) {
expFlags.addAll(flag.split(","));
}
}
Verbosity verbosity = Verbosity.parseArgument(verbosityLevel);
return new CompilerOptions()
..fileSystem = fileSystem
..target = new VmTarget(new TargetFlags(
enableNullSafety: nullSafety == kNullSafetyOptionStrong))
..packagesFileUri = packagesUri
..sdkSummary = platformKernelPath
..verbose = verbose
..omitPlatform = true
..explicitExperimentalFlags = parseExperimentalFlags(
parseExperimentalArguments(expFlags), onError: (msg) {
errorsPlain.add(msg);
errorsColorized.add(msg);
})
..environmentDefines = new EnvironmentMap()
..nnbdMode = (nullSafety == kNullSafetyOptionStrong)
? NnbdMode.Strong
: NnbdMode.Weak
..onDiagnostic = (DiagnosticMessage message) {
bool printToStdErr = false;
bool printToStdOut = false;
switch (message.severity) {
case Severity.error:
case Severity.internalProblem:
// TODO(sigmund): support emitting code with errors as long as they
// are handled in the generated code.
printToStdErr = false; // errors are printed by VM
errorsPlain.addAll(message.plainTextFormatted);
errorsColorized.addAll(message.ansiFormatted);
break;
case Severity.warning:
printToStdErr = true;
break;
case Severity.info:
printToStdOut = true;
break;
case Severity.context:
case Severity.ignored:
throw "Unexpected severity: ${message.severity}";
}
if (Verbosity.shouldPrint(verbosity, message)) {
if (printToStdErr) {
printDiagnosticMessage(message, stderr.writeln);
} else if (printToStdOut) {
printDiagnosticMessage(message, stdout.writeln);
}
}
}
..invocationModes = InvocationMode.parseArguments(invocationModes)
..verbosity = verbosity;
}
abstract class Compiler {
final int isolateGroupId;
final FileSystem fileSystem;
final Uri? platformKernelPath;
final bool enableAsserts;
final int nullSafety;
final List<String>? experimentalFlags;
final String? packageConfig;
final String invocationModes;
final String verbosityLevel;
// Code coverage and hot reload are only supported by incremental compiler,
// which is used if vm-service is enabled.
final bool supportCodeCoverage;
final bool supportHotReload;
final List<String> errorsPlain = <String>[];
final List<String> errorsColorized = <String>[];
late final CompilerOptions options;
Compiler(this.isolateGroupId, this.fileSystem, this.platformKernelPath,
{this.enableAsserts: false,
this.nullSafety: kNullSafetyOptionUnspecified,
this.experimentalFlags: null,
this.supportCodeCoverage: false,
this.supportHotReload: false,
this.packageConfig: null,
this.invocationModes: '',
this.verbosityLevel: Verbosity.defaultValue}) {
Uri? packagesUri = null;
final packageConfig = this.packageConfig ?? Platform.packageConfig;
if (packageConfig != null) {
packagesUri = Uri.parse(packageConfig);
}
if (verbose) {
print("DFE: Platform.packageConfig: ${Platform.packageConfig}");
print("DFE: packagesUri: ${packagesUri}");
print("DFE: Platform.resolvedExecutable: ${Platform.resolvedExecutable}");
print("DFE: platformKernelPath: ${platformKernelPath}");
}
options = setupCompilerOptions(
fileSystem,
platformKernelPath,
enableAsserts,
nullSafety,
experimentalFlags,
packagesUri,
errorsPlain,
errorsColorized,
invocationModes,
verbosityLevel);
}
Future<CompilerResult> compile(Uri script) {
return runWithPrintToStderr(() async {
final CompilerResult compilerResult = await compileInternal(script);
final Component? component = compilerResult.component;
if (errorsPlain.isEmpty) {
// Record dependencies only if compilation was error free.
_recordDependencies(isolateGroupId, component, options.packagesFileUri);
}
return compilerResult;
});
}
Future<CompilerResult> compileInternal(Uri script);
}
class CompilerResult {
final Component? component;
/// Set of libraries loaded from .dill, with or without the SDK depending on
/// the compilation settings.
final Set<Library> loadedLibraries;
final ClassHierarchy? classHierarchy;
final CoreTypes? coreTypes;
CompilerResult(this.component, this.loadedLibraries, this.classHierarchy,
this.coreTypes);
}
// Environment map which looks up environment defines in the VM environment
// at runtime.
// TODO(askesc): This is a temporary hack to get hold of the environment during
// JIT compilation. We use a lazy map accessing the VM runtime environment using
// new String.fromEnvironment, since the VM currently does not support providing
// the full (isolate specific) environment as a finite, static map.
class EnvironmentMap extends UnmodifiableMapBase<String, String> {
@override
bool containsKey(Object? key) {
return key is String && new bool.hasEnvironment(key);
}
@override
String? operator [](Object? key) {
// The fromEnvironment constructor is specified to throw when called using
// new. However, the VM implementation actually looks up the given name in
// the environment.
if (containsKey(key)) {
return new String.fromEnvironment(key as String);
}
return null;
}
@override
get keys => throw "Environment map iteration not supported";
}
class FileSink implements Sink<List<int>> {
MemoryFileSystemEntity entityForUri;
List<int> bytes = <int>[];
FileSink(this.entityForUri);
@override
void add(List<int> data) {
bytes.addAll(data);
}
@override
void close() {
this.entityForUri.writeAsBytesSync(bytes);
}
}
class IncrementalCompilerWrapper extends Compiler {
IncrementalCompiler? generator;
IncrementalCompilerWrapper(
int isolateGroupId, FileSystem fileSystem, Uri? platformKernelPath,
{bool enableAsserts: false,
int nullSafety: kNullSafetyOptionUnspecified,
List<String>? experimentalFlags,
String? packageConfig,
String invocationModes: '',
String verbosityLevel: Verbosity.defaultValue})
: super(isolateGroupId, fileSystem, platformKernelPath,
enableAsserts: enableAsserts,
nullSafety: nullSafety,
experimentalFlags: experimentalFlags,
supportHotReload: true,
supportCodeCoverage: true,
packageConfig: packageConfig,
invocationModes: invocationModes,
verbosityLevel: verbosityLevel);
factory IncrementalCompilerWrapper.forExpressionCompilationOnly(
Component component,
int isolateGroupId,
FileSystem fileSystem,
Uri? platformKernelPath,
{bool enableAsserts: false,
List<String>? experimentalFlags,
String? packageConfig,
String invocationModes: ''}) {
IncrementalCompilerWrapper result = IncrementalCompilerWrapper(
isolateGroupId, fileSystem, platformKernelPath,
enableAsserts: enableAsserts,
experimentalFlags: experimentalFlags,
packageConfig: packageConfig,
invocationModes: invocationModes);
result.generator = new IncrementalCompiler.forExpressionCompilationOnly(
component,
result.options,
component.mainMethod!.enclosingLibrary.fileUri);
return result;
}
@override
Future<CompilerResult> compileInternal(Uri script) async {
final generator = this.generator ??= IncrementalCompiler(options, script);
errorsPlain.clear();
errorsColorized.clear();
final compilerResult = await generator.compile(entryPoint: script);
final component = compilerResult.component;
return new CompilerResult(component, const {},
compilerResult.classHierarchy, compilerResult.coreTypes);
}
void accept() => generator!.accept();
void invalidate(Uri uri) => generator!.invalidate(uri);
Future<IncrementalCompilerWrapper> clone(int isolateGroupId) async {
IncrementalCompilerWrapper clone = IncrementalCompilerWrapper(
isolateGroupId, fileSystem, platformKernelPath,
enableAsserts: enableAsserts,
nullSafety: nullSafety,
experimentalFlags: experimentalFlags,
packageConfig: packageConfig,
invocationModes: invocationModes);
final generator = this.generator!;
// TODO(VM TEAM): This does not seem safe. What if cloning while having
// pending deltas for instance?
generator.resetDeltaState();
IncrementalCompilerResult compilerResult = await generator.compile();
Component fullComponent = compilerResult.component;
// Assume fileSystem is HybridFileSystem because that is the setup where
// clone should be used for.
MemoryFileSystem memoryFileSystem = (fileSystem as HybridFileSystem).memory;
String filename = 'full-component-$isolateGroupId.dill';
Sink<List<int>> sink =
FileSink(memoryFileSystem.entityForUri(Uri.file(filename)));
new BinaryPrinter(sink).writeComponentFile(fullComponent);
sink.close();
clone.generator = new IncrementalCompiler(options, generator.entryPoint,
initializeFromDillUri: Uri.file(filename));
return clone;
}
}
class SingleShotCompilerWrapper extends Compiler {
final bool requireMain;
SingleShotCompilerWrapper(
int isolateGroupId, FileSystem fileSystem, Uri platformKernelPath,
{this.requireMain: false,
bool enableAsserts: false,
int nullSafety: kNullSafetyOptionUnspecified,
List<String>? experimentalFlags,
String? packageConfig,
String invocationModes: '',
String verbosityLevel: Verbosity.defaultValue})
: super(isolateGroupId, fileSystem, platformKernelPath,
enableAsserts: enableAsserts,
nullSafety: nullSafety,
experimentalFlags: experimentalFlags,
packageConfig: packageConfig,
invocationModes: invocationModes,
verbosityLevel: verbosityLevel);
@override
Future<CompilerResult> compileInternal(Uri script) async {
final fe.CompilerResult? compilerResult = requireMain
? await kernelForProgram(script, options)
: await kernelForModule([script], options);
if (compilerResult == null) {
return CompilerResult(null, const {}, null, null);
}
Set<Library> loadedLibraries = createLoadedLibrariesSet(
compilerResult.loadedComponents, compilerResult.sdkComponent,
includePlatform: !options.omitPlatform);
return new CompilerResult(compilerResult.component, loadedLibraries,
compilerResult.classHierarchy, compilerResult.coreTypes);
}
}
final Map<int, IncrementalCompilerWrapper> isolateCompilers = {};
final Map<int, List<Uri>> isolateDependencies = {};
final Map<int, _ExpressionCompilationFromDillSettings> isolateLoadNotifies = {};
IncrementalCompilerWrapper? lookupIncrementalCompiler(int isolateGroupId) {
return isolateCompilers[isolateGroupId];
}
Future<Compiler> lookupOrBuildNewIncrementalCompiler(int isolateGroupId,
List sourceFiles, Uri platformKernelPath, List<int>? platformKernel,
{bool enableAsserts: false,
int nullSafety: kNullSafetyOptionUnspecified,
List<String>? experimentalFlags,
String? packageConfig,
String? multirootFilepaths,
String? multirootScheme,
String invocationModes: '',
String verbosityLevel: Verbosity.defaultValue}) async {
IncrementalCompilerWrapper? compiler =
lookupIncrementalCompiler(isolateGroupId);
if (compiler != null) {
updateSources(compiler, sourceFiles);
invalidateSources(compiler, sourceFiles);
} else {
// This is how identify scenario where child isolate hot reload requests
// requires setting up actual compiler first: non-empty sourceFiles list has
// no actual content specified for the source file.
if (sourceFiles.isNotEmpty && sourceFiles[1] == null) {
// Just use first compiler that should represent main isolate as a source for cloning.
var source = isolateCompilers.entries.first;
compiler = await source.value.clone(isolateGroupId);
} else {
FileSystem fileSystem = _buildFileSystem(
sourceFiles, platformKernel, multirootFilepaths, multirootScheme);
// TODO(aam): IncrementalCompilerWrapper instance created below have to be
// destroyed when corresponding isolate is shut down. To achieve that kernel
// isolate needs to receive a message indicating that particular
// isolate was shut down. Message should be handled here in this script.
compiler = new IncrementalCompilerWrapper(
isolateGroupId, fileSystem, platformKernelPath,
enableAsserts: enableAsserts,
nullSafety: nullSafety,
experimentalFlags: experimentalFlags,
packageConfig: packageConfig,
invocationModes: invocationModes,
verbosityLevel: verbosityLevel);
}
isolateCompilers[isolateGroupId] = compiler;
}
return compiler;
}
void updateSources(IncrementalCompilerWrapper compiler, List sourceFiles) {
final bool hasMemoryFS = compiler.fileSystem is HybridFileSystem;
if (sourceFiles.isNotEmpty) {
final FileSystem fs = compiler.fileSystem;
for (int i = 0; i < sourceFiles.length ~/ 2; i++) {
Uri uri = Uri.parse(sourceFiles[i * 2]);
List<int>? source = sourceFiles[i * 2 + 1];
// The source is only provided by unit tests and is normally empty.
// Don't add an entry for the uri so the compiler will fallback to the
// real file system for the updated source.
if (hasMemoryFS && source != null) {
(fs as HybridFileSystem)
.memory
.entityForUri(uri)
.writeAsBytesSync(source);
}
}
}
}
void invalidateSources(IncrementalCompilerWrapper compiler, List sourceFiles) {
if (sourceFiles.isNotEmpty) {
for (int i = 0; i < sourceFiles.length ~/ 2; i++) {
compiler.invalidate(Uri.parse(sourceFiles[i * 2]));
}
}
}
// Process a request from the runtime. See KernelIsolate::CompileToKernel in
// kernel_isolate.cc and Loader::SendKernelRequest in loader.cc.
Future _processExpressionCompilationRequest(request) async {
final SendPort port = request[1];
final int isolateGroupId = request[2];
final dynamic dart_platform_kernel = request[3];
final String expression = request[4];
final List<String> definitions = request[5].cast<String>();
final List<String> typeDefinitions = request[6].cast<String>();
final String libraryUri = request[7];
final String? klass = request[8];
final String? method = request[9];
final bool isStatic = request[10];
final List<List<int>> dillData = request[11].cast<List<int>>();
final int blobLoadCount = request[12];
final bool enableAsserts = request[13];
final List<String>? experimentalFlags =
request[14] != null ? request[14].cast<String>() : null;
IncrementalCompilerWrapper? compiler = isolateCompilers[isolateGroupId];
_ExpressionCompilationFromDillSettings? isolateLoadDillData =
isolateLoadNotifies[isolateGroupId];
if (isolateLoadDillData != null) {
// Check if we can reuse the compiler.
if (isolateLoadDillData.blobLoadCount != blobLoadCount ||
isolateLoadDillData.prevDillCount != dillData.length) {
isolateCompilers.remove(isolateGroupId);
compiler = null;
}
}
if (compiler == null) {
if (dillData.isNotEmpty) {
if (verbose) {
print("DFE: Initializing compiler from ${dillData.length} dill files");
}
isolateLoadNotifies[isolateGroupId] =
new _ExpressionCompilationFromDillSettings(
blobLoadCount, dillData.length);
// Create Component initialized from the bytes.
Component component = new Component();
// First try to just load all "dillData". This *might* include the
// platform (and we might have the (same) platform both here and in
// dart_platform_kernel).
for (List<int> bytes in dillData) {
// TODO(jensj): There might be an issue if main has changed.
new BinaryBuilderWithMetadata(bytes, alwaysCreateNewNamedNodes: true)
.readComponent(component);
}
// Check if the loaded component has the platform.
// If it does not, try to load from dart_platform_kernel or from file.
bool foundDartCore = false;
for (Library library in component.libraries) {
if (library.importUri.scheme == "dart" &&
library.importUri.path == "core" &&
!library.isSynthetic) {
foundDartCore = true;
break;
}
}
if (!foundDartCore) {
List<int> platformKernel;
if (dart_platform_kernel is List<int>) {
platformKernel = dart_platform_kernel;
} else {
final Uri platformUri = computePlatformBinariesLocation()
.resolve('vm_platform_strong.dill');
final File platformFile = new File.fromUri(platformUri);
if (platformFile.existsSync()) {
platformKernel = platformFile.readAsBytesSync();
} else {
port.send(new CompilationResult.errors(
["No platform found to initialize incremental compiler."],
null)
.toResponse());
return;
}
}
new BinaryBuilderWithMetadata(platformKernel,
alwaysCreateNewNamedNodes: true)
.readComponent(component);
}
FileSystem fileSystem =
_buildFileSystem([dotPackagesFile, <int>[]], null, null, null);
// TODO(aam): IncrementalCompilerWrapper instance created below have to be
// destroyed when corresponding isolate is shut down. To achieve that
// kernel isolate needs to receive a message indicating that particular
// isolate was shut down. Message should be handled here in this script.
try {
compiler = new IncrementalCompilerWrapper.forExpressionCompilationOnly(
component, isolateGroupId, fileSystem, null,
enableAsserts: enableAsserts,
experimentalFlags: experimentalFlags,
packageConfig: dotPackagesFile);
isolateCompilers[isolateGroupId] = compiler;
await compiler.compile(
component.mainMethod?.enclosingLibrary.importUri ??
component.libraries.last.importUri);
} catch (e) {
port.send(new CompilationResult.errors([
"Error when trying to create a compiler for expression compilation: "
"'$e'."
], null)
.toResponse());
return;
}
}
}
if (compiler == null) {
port.send(new CompilationResult.errors(
["No incremental compiler available for this isolate."], null)
.toResponse());
return;
}
compiler.errorsPlain.clear();
compiler.errorsColorized.clear();
CompilationResult result;
try {
Procedure? procedure = await compiler.generator!.compileExpression(
expression,
definitions,
typeDefinitions,
libraryUri,
klass,
method,
isStatic);
if (procedure == null) {
port.send(
new CompilationResult.errors(["Invalid scope."], null).toResponse());
return;
}
assert(compiler.errorsPlain.length == compiler.errorsColorized.length);
// Any error will be printed verbatim in observatory, so we always use the
// plain version (i.e. the one without ANSI escape codes in it).
if (compiler.errorsPlain.isNotEmpty) {
// TODO(sigmund): the compiler prints errors to the console, so we
// shouldn't print those messages again here.
result = new CompilationResult.errors(compiler.errorsPlain, null);
} else {
Component component = createExpressionEvaluationComponent(procedure);
result = new CompilationResult.ok(serializeComponent(component));
}
} catch (error, stack) {
result = new CompilationResult.crash(error, stack);
}
port.send(result.toResponse());
}
void _recordDependencies(
int isolateGroupId, Component? component, Uri? packageConfig) {
final dependencies = isolateDependencies[isolateGroupId] ??= <Uri>[];
if (component != null) {
for (var lib in component.libraries) {
if (lib.importUri.scheme == "dart") continue;
dependencies.add(lib.fileUri);
for (var part in lib.parts) {
final fileUri = lib.fileUri.resolve(part.partUri);
if (fileUri.scheme != "" && fileUri.scheme != "file") {
// E.g. part 'package:foo/foo.dart';
// Maybe the front end should resolve this?
continue;
}
dependencies.add(fileUri);
}
}
}
if (packageConfig != null) {
dependencies.add(packageConfig);
}
}
String _escapeDependency(Uri uri) {
return uri.toFilePath().replaceAll("\\", "\\\\").replaceAll(" ", "\\ ");
}
Uint8List _serializeDependencies(List<Uri> uris) {
return utf8.encode(uris.map(_escapeDependency).join(" ")) as Uint8List;
}
Future _processListDependenciesRequest(
SendPort port, int isolateGroupId) async {
final List<Uri> dependencies = isolateDependencies[isolateGroupId] ?? <Uri>[];
CompilationResult result;
try {
result = new CompilationResult.ok(_serializeDependencies(dependencies));
} catch (error, stack) {
result = new CompilationResult.crash(error, stack);
}
port.send(result.toResponse());
}
Future _processIsolateShutdownNotification(request) async {
final int isolateGroupId = request[1];
isolateCompilers.remove(isolateGroupId);
isolateDependencies.remove(isolateGroupId);
isolateLoadNotifies.remove(isolateGroupId);
}
Future _processLoadRequest(request) async {
if (verbose) {
for (int i = 0; i < request.length; i++) {
var part = request[i];
String partToString;
if (part is List && part.isNotEmpty) {
// Assume this is large and printing all of it takes a lot of time.
StringBuffer sb = new StringBuffer();
String prepend = "[";
for (int j = 0; j < part.length; j++) {
sb.write(prepend);
sb.write(part[j]);
prepend = ", ";
if (sb.length > 256) break;
}
sb.write("]");
partToString = sb.toString();
} else {
partToString = part.toString();
}
if (partToString.length > 256) {
partToString = partToString.substring(0, 255) + "...";
}
print("DFE: request[$i]: $partToString");
}
}
int tag = request[0];
if (tag == kCompileExpressionTag) {
await _processExpressionCompilationRequest(request);
return;
}
if (tag == kNotifyIsolateShutdownTag) {
await _processIsolateShutdownNotification(request);
return;
}
final SendPort port = request[1];
final int isolateGroupId = request[7];
if (tag == kListDependenciesTag) {
await _processListDependenciesRequest(port, isolateGroupId);
return;
}
final String? inputFileUri = request[2];
final Uri? script =
inputFileUri != null ? Uri.base.resolve(inputFileUri) : null;
final bool incremental = request[4];
final bool snapshot = request[5];
final int nullSafety = request[6];
final List sourceFiles = request[8];
final bool enableAsserts = request[9];
final List<String>? experimentalFlags =
request[10] != null ? request[10].cast<String>() : null;
final String? packageConfig = request[11];
final String? multirootFilepaths = request[12];
final String? multirootScheme = request[13];
final String? workingDirectory = request[14];
final String verbosityLevel = request[15];
Uri platformKernelPath;
List<int>? platformKernel = null;
if (request[3] is String) {
platformKernelPath = Uri.base.resolveUri(new Uri.file(request[3]));
} else if (request[3] is List<int>) {
platformKernelPath = Uri.parse(platformKernelFile);
platformKernel = request[3];
} else {
platformKernelPath =
computePlatformBinariesLocation().resolve('vm_platform_strong.dill');
}
final String invocationModes = snapshot ? 'compile' : '';
Compiler? compiler;
// Update the in-memory file system with the provided sources. Currently, only
// unit tests compile sources that are not on the file system, so this can only
// happen during unit tests.
if (tag == kUpdateSourcesTag) {
assert(incremental,
"Incremental compiler required for use of 'kUpdateSourcesTag'");
compiler = lookupIncrementalCompiler(isolateGroupId);
if (compiler == null) {
port.send(new CompilationResult.errors(
["No incremental compiler available for this isolate."], null)
.toResponse());
return;
}
updateSources(compiler as IncrementalCompilerWrapper, sourceFiles);
port.send(new CompilationResult.ok(null).toResponse());
return;
} else if (tag == kAcceptTag) {
assert(
incremental, "Incremental compiler required for use of 'kAcceptTag'");
compiler = lookupIncrementalCompiler(isolateGroupId);
// There are unit tests that invoke the IncrementalCompiler directly and
// request a reload, meaning that we won't have a compiler for this isolate.
if (compiler != null) {
(compiler as IncrementalCompilerWrapper).accept();
}
port.send(new CompilationResult.ok(null).toResponse());
return;
} else if (tag == kDetectNullabilityTag) {
FileSystem fileSystem = _buildFileSystem(
sourceFiles, platformKernel, multirootFilepaths, multirootScheme);
Uri? packagesUri = null;
final packageConfigWithDefault = packageConfig ?? Platform.packageConfig;
if (packageConfigWithDefault != null) {
packagesUri = Uri.parse(packageConfigWithDefault);
}
if (packagesUri != null && packagesUri.scheme == '') {
// Script does not have a scheme, assume that it is a path,
// resolve it against the working directory.
packagesUri = Uri.directory(workingDirectory!).resolveUri(packagesUri);
}
final List<String> errorsPlain = <String>[];
final List<String> errorsColorized = <String>[];
var options = setupCompilerOptions(
fileSystem,
platformKernelPath,
false,
nullSafety,
experimentalFlags,
packagesUri,
errorsPlain,
errorsColorized,
invocationModes,
verbosityLevel);
// script should only be null for kUpdateSourcesTag.
await autoDetectNullSafetyMode(script!, options);
bool value = options.nnbdMode == NnbdMode.Strong;
port.send(new CompilationResult.nullSafety(value).toResponse());
return;
}
// script should only be null for kUpdateSourcesTag.
assert(script != null);
// TODO(aam): There should be no need to have an option to choose
// one compiler or another. We should always use an incremental
// compiler as its functionality is a super set of the other one. We need to
// watch the performance though.
if (incremental) {
compiler = await lookupOrBuildNewIncrementalCompiler(
isolateGroupId, sourceFiles, platformKernelPath, platformKernel,
enableAsserts: enableAsserts,
nullSafety: nullSafety,
experimentalFlags: experimentalFlags,
packageConfig: packageConfig,
multirootFilepaths: multirootFilepaths,
multirootScheme: multirootScheme,
invocationModes: invocationModes,
verbosityLevel: verbosityLevel);
} else {
FileSystem fileSystem = _buildFileSystem(
sourceFiles, platformKernel, multirootFilepaths, multirootScheme);
compiler = new SingleShotCompilerWrapper(
isolateGroupId, fileSystem, platformKernelPath,
requireMain: false,
enableAsserts: enableAsserts,
nullSafety: nullSafety,
experimentalFlags: experimentalFlags,
packageConfig: packageConfig,
invocationModes: invocationModes,
verbosityLevel: verbosityLevel);
}
CompilationResult result;
try {
if (verbose) {
print("DFE: scriptUri: ${script}");
}
CompilerResult compilerResult = await compiler.compile(script!);
Set<Library> loadedLibraries = compilerResult.loadedLibraries;
assert(compiler.errorsPlain.length == compiler.errorsColorized.length);
// http://dartbug.com/45137
// enableColors calls `stdout.supportsAnsiEscapes` which - on Windows -
// does something with line endings. To avoid this when no error
// messages are do be printed anyway, we are carefull not to call it unless
// neccessary.
if (compiler.errorsColorized.isNotEmpty) {
final List<String> errors =
(enableColors) ? compiler.errorsColorized : compiler.errorsPlain;
final component = compilerResult.component;
if (component != null) {
result = new CompilationResult.errors(
errors,
serializeComponent(component,
filter: (lib) => !loadedLibraries.contains(lib)));
} else {
result = new CompilationResult.errors(errors, null);
}
} else {
// We serialize the component excluding vm_platform.dill because the VM has
// these sources built-in. Everything loaded as a summary in
// [kernelForProgram] is marked `external`, so we can use that bit to
// decide what to exclude.
result = new CompilationResult.ok(serializeComponent(
compilerResult.component!,
filter: (lib) => !loadedLibraries.contains(lib)));
}
} catch (error, stack) {
result = new CompilationResult.crash(error, stack);
}
if (verbose) print("DFE:> ${result}");
if (tag == kTrainTag) {
// In training mode make sure to read the sdk a few more times...
ProcessedOptions p = new ProcessedOptions(options: compiler.options);
final bytes = (await p.loadSdkSummaryBytes())!;
for (int i = 0; i < 5; i++) {
p.loadComponent(bytes, null);
}
if (result.status != Status.ok) {
tag = -tag;
}
port.send([tag, inputFileUri, inputFileUri, null, result.payload]);
} else if (tag == kCompileTag) {
port.send(result.toResponse());
} else {
port.send([
-tag,
inputFileUri,
inputFileUri,
null,
new CompilationResult.errors(<String>["unknown tag"], null).payload
]);
}
}
/// Creates a file system containing the files specified in [sourceFiles] and
/// that delegates to the underlying file system for any other file request.
/// The [sourceFiles] list interleaves file name string and
/// raw file content Uint8List.
///
/// The result can be used instead of StandardFileSystem.instance by the
/// frontend.
FileSystem _buildFileSystem(List sourceFiles, List<int>? platformKernel,
String? multirootFilepaths, String? multirootScheme) {
FileSystem fileSystem = new HttpAwareFileSystem(StandardFileSystem.instance);
if (!sourceFiles.isEmpty || platformKernel != null) {
MemoryFileSystem memoryFileSystem =
new MemoryFileSystem(Uri.parse('file:///'));
for (int i = 0; i < sourceFiles.length ~/ 2; i++) {
memoryFileSystem
.entityForUri(Uri.parse(sourceFiles[i * 2]))
.writeAsBytesSync(sourceFiles[i * 2 + 1]);
}
if (platformKernel != null) {
memoryFileSystem
.entityForUri(Uri.parse(platformKernelFile))
.writeAsBytesSync(platformKernel);
}
fileSystem = new HybridFileSystem(memoryFileSystem, fileSystem);
}
if (multirootFilepaths != null) {
List<Uri> list = multirootFilepaths
.split(',')
.map((String s) => Uri.base.resolveUri(new Uri.file(s)))
.toList();
fileSystem = new MultiRootFileSystem(
multirootScheme ?? "org-dartlang-root", list, fileSystem);
}
return fileSystem;
}
train(String scriptUri, String? platformKernelPath) async {
// Train on program asked to train on.
await trainInternal(scriptUri, platformKernelPath);
// Also train a few times on a hello-world program to make sure we exercise
// the startup sequence.
Directory tmpDir =
Directory.systemTemp.createTempSync("kernel_service_train");
File helloDart = new File.fromUri(tmpDir.uri.resolve("hello.dart"));
helloDart.writeAsStringSync("""
main() {
print("Hello, World!");
}
""");
try {
for (int i = 0; i < 10; i++) {
await trainInternal(helloDart.uri.toString(), platformKernelPath);
}
} finally {
tmpDir.deleteSync(recursive: true);
}
}
Future trainInternal(String scriptUri, String? platformKernelPath) async {
var tag = kTrainTag;
var responsePort = new RawReceivePort();
responsePort.handler = (response) {
if (response[0] == tag) {
// Success.
responsePort.close();
} else if (response[0] == -tag) {
// Compilation error.
throw response[4];
} else {
throw "Unexpected response: $response";
}
};
var request = [
tag,
responsePort.sendPort,
scriptUri,
platformKernelPath,
false /* incremental */,
false /* snapshot */,
kNullSafetyOptionUnspecified /* null safety */,
1 /* isolateGroupId chosen randomly */,
[] /* source files */,
false /* enable asserts */,
null /* experimental_flags */,
null /* package_config */,
null /* multirootFilepaths */,
null /* multirootScheme */,
null /* original working directory */,
'all' /* CFE logging mode */,
];
await _processLoadRequest(request);
}
main([args]) {
if ((args?.length ?? 0) > 1 && args[0] == '--train') {
// This entry point is used when creating an app snapshot.
// It takes the following extra arguments:
// 1) Script to compile.
// 2) Optional platform kernel path.
int argIndex = 1;
final String script = args[argIndex++];
final String? platform = (argIndex < args.length) ? args[argIndex] : null;
train(script, platform);
} else {
// Entry point for the Kernel isolate.
return new RawReceivePort()..handler = _processLoadRequest;
}
}
/// Compilation status codes.
///
/// Note: The [index] property of these constants must match
/// `Dart_KernelCompilationStatus` in
/// [dart_api.h](../../../../runtime/include/dart_api.h).
enum Status {
/// Compilation was successful.
ok,
/// Compilation failed with a compile time error.
error,
/// Compiler crashed.
crash,
}
abstract class CompilationResult {
CompilationResult._();
factory CompilationResult.ok(Uint8List? bytes) = _CompilationOk;
factory CompilationResult.nullSafety(bool val) = _CompilationNullSafety;
factory CompilationResult.errors(List<String> errors, Uint8List? bytes) =
_CompilationError;
factory CompilationResult.crash(Object exception, StackTrace stack) =
_CompilationCrash;
Status get status;
get payload;
List toResponse() => [status.index, payload];
}
class _CompilationOk extends CompilationResult {
final Uint8List? bytes;
_CompilationOk(this.bytes) : super._() {
if (dumpKernel) {
final bytes = this.bytes;
if (bytes != null) {
_debugDumpKernel(bytes);
}
}
}
@override
Status get status => Status.ok;
@override
get payload => bytes;
String toString() => "_CompilationOk(${bytes?.length ?? 0} bytes)";
}
class _CompilationNullSafety extends CompilationResult {
final bool _null_safety;
_CompilationNullSafety(this._null_safety) : super._() {}
@override
Status get status => Status.ok;
@override
get payload => _null_safety;
String toString() => "_CompilationNullSafety($_null_safety)";
}
abstract class _CompilationFail extends CompilationResult {
_CompilationFail() : super._();
String get errorString;
@override
get payload => errorString;
}
class _CompilationError extends _CompilationFail {
final Uint8List? bytes;
final List<String> errors;
_CompilationError(this.errors, this.bytes);
@override
Status get status => Status.error;
@override
String get errorString => errors.take(10).join('\n');
String toString() => "_CompilationError(${errorString})";
List toResponse() => [status.index, payload, bytes];
}
class _CompilationCrash extends _CompilationFail {
final Object exception;
final StackTrace stack;
_CompilationCrash(this.exception, this.stack);
@override
Status get status => Status.crash;
@override
String get errorString => "${exception}\n${stack}";
String toString() => "_CompilationCrash(${errorString})";
}
Future<T> runWithPrintToStderr<T>(Future<T> f()) {
return runZoned(() => new Future<T>(f),
zoneSpecification: new ZoneSpecification(
print: (_1, _2, _3, String line) => stderr.writeln(line)));
}
int _debugDumpCounter = 0;
void _debugDumpKernel(Uint8List bytes) {
new File('kernel_service.tmp${_debugDumpCounter++}.dill')
.writeAsBytesSync(bytes);
}
class _ExpressionCompilationFromDillSettings {
int blobLoadCount;
int prevDillCount;
_ExpressionCompilationFromDillSettings(
this.blobLoadCount, this.prevDillCount);
}