| // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| /// This is an interface to the Dart Kernel parser and Kernel binary generator. |
| /// |
| /// It is used by the kernel-isolate to load Dart source code and generate |
| /// Kernel binary format. |
| /// |
| /// This is either invoked as the root script of the Kernel isolate when used |
| /// as a part of |
| /// |
| /// dart --dfe=pkg/vm/bin/kernel_service.dart ... |
| /// |
| /// invocation or it is invoked as a standalone script to perform training for |
| /// the app-jit snapshot |
| /// |
| /// dart pkg/vm/bin/kernel_service.dart --train <source-file> |
| /// |
| /// |
| |
| import 'dart:async' show Future, ZoneSpecification, runZoned; |
| import 'dart:collection' show UnmodifiableMapBase; |
| import 'dart:convert' show utf8; |
| import 'dart:io' |
| show Directory, File, Platform, stderr, stdout |
| hide FileSystemEntity; |
| import 'dart:isolate'; |
| import 'dart:typed_data' show Uint8List; |
| |
| import 'package:build_integration/file_system/multi_root.dart'; |
| import 'package:front_end/src/api_prototype/front_end.dart' as fe |
| show CompilerResult; |
| import 'package:front_end/src/api_prototype/memory_file_system.dart'; |
| import 'package:front_end/src/api_unstable/vm.dart'; |
| import 'package:kernel/binary/ast_to_binary.dart'; |
| import 'package:kernel/binary/ast_from_binary.dart' |
| show BinaryBuilderWithMetadata; |
| import 'package:kernel/class_hierarchy.dart' show ClassHierarchy; |
| import 'package:kernel/core_types.dart' show CoreTypes; |
| import 'package:kernel/kernel.dart' show Component, Library, Procedure; |
| import 'package:kernel/target/targets.dart' show TargetFlags; |
| import 'package:vm/incremental_compiler.dart'; |
| import 'package:vm/kernel_front_end.dart' |
| show autoDetectNullSafetyMode, createLoadedLibrariesSet; |
| import 'package:vm/http_filesystem.dart'; |
| import 'package:vm/target/vm.dart' show VmTarget; |
| |
| final bool verbose = new bool.fromEnvironment('DFE_VERBOSE'); |
| final bool dumpKernel = new bool.fromEnvironment('DFE_DUMP_KERNEL'); |
| const String platformKernelFile = 'virtual_platform_kernel.dill'; |
| const String dotPackagesFile = '.packages'; |
| |
| // NOTE: Any changes to these tags need to be reflected in kernel_isolate.cc |
| // Tags used to indicate different requests to the dart frontend. |
| // |
| // Current tags include the following: |
| // 0 - Perform normal compilation. |
| // 1 - Update in-memory file system with in-memory sources (used by tests). |
| // 2 - Accept last compilation result. |
| // 3 - APP JIT snapshot training run for kernel_service. |
| // 4 - Compile an individual expression in some context (for debugging |
| // purposes). |
| // 5 - List program dependencies (for creating depfiles) |
| // 6 - Isolate shutdown that potentially should result in compiler cleanup. |
| const int kCompileTag = 0; |
| const int kUpdateSourcesTag = 1; |
| const int kAcceptTag = 2; |
| const int kTrainTag = 3; |
| const int kCompileExpressionTag = 4; |
| const int kListDependenciesTag = 5; |
| const int kNotifyIsolateShutdownTag = 6; |
| const int kDetectNullabilityTag = 7; |
| |
| bool allowDartInternalImport = false; |
| |
| // Null Safety command line options |
| // |
| // Note: The values of these constants must match the |
| // values of flag sound_null_safety in ../../../../runtime/vm/flag_list.h. |
| // 0 - No --[no-]sound-null-safety option specified on the command line. |
| // 1 - '--no-sound-null-safety' specified on the command line. |
| // 2 - '--sound-null-safety' option specified on the command line. |
| const int kNullSafetyOptionUnspecified = 0; |
| const int kNullSafetyOptionWeak = 1; |
| const int kNullSafetyOptionStrong = 2; |
| |
| CompilerOptions setupCompilerOptions( |
| FileSystem fileSystem, |
| Uri? platformKernelPath, |
| bool enableAsserts, |
| int nullSafety, |
| List<String>? experimentalFlags, |
| Uri? packagesUri, |
| List<String> errorsPlain, |
| List<String> errorsColorized, |
| String invocationModes, |
| String verbosityLevel) { |
| final expFlags = <String>[]; |
| if (experimentalFlags != null) { |
| for (String flag in experimentalFlags) { |
| expFlags.addAll(flag.split(",")); |
| } |
| } |
| |
| Verbosity verbosity = Verbosity.parseArgument(verbosityLevel); |
| return new CompilerOptions() |
| ..fileSystem = fileSystem |
| ..target = new VmTarget(new TargetFlags( |
| enableNullSafety: nullSafety == kNullSafetyOptionStrong)) |
| ..packagesFileUri = packagesUri |
| ..sdkSummary = platformKernelPath |
| ..verbose = verbose |
| ..omitPlatform = true |
| ..explicitExperimentalFlags = parseExperimentalFlags( |
| parseExperimentalArguments(expFlags), onError: (msg) { |
| errorsPlain.add(msg); |
| errorsColorized.add(msg); |
| }) |
| ..environmentDefines = new EnvironmentMap() |
| ..nnbdMode = (nullSafety == kNullSafetyOptionStrong) |
| ? NnbdMode.Strong |
| : NnbdMode.Weak |
| ..onDiagnostic = (DiagnosticMessage message) { |
| bool printToStdErr = false; |
| bool printToStdOut = false; |
| switch (message.severity) { |
| case Severity.error: |
| case Severity.internalProblem: |
| // TODO(sigmund): support emitting code with errors as long as they |
| // are handled in the generated code. |
| printToStdErr = false; // errors are printed by VM |
| errorsPlain.addAll(message.plainTextFormatted); |
| errorsColorized.addAll(message.ansiFormatted); |
| break; |
| case Severity.warning: |
| printToStdErr = true; |
| break; |
| case Severity.info: |
| printToStdOut = true; |
| break; |
| case Severity.context: |
| case Severity.ignored: |
| throw "Unexpected severity: ${message.severity}"; |
| } |
| if (Verbosity.shouldPrint(verbosity, message)) { |
| if (printToStdErr) { |
| printDiagnosticMessage(message, stderr.writeln); |
| } else if (printToStdOut) { |
| printDiagnosticMessage(message, stdout.writeln); |
| } |
| } |
| } |
| ..invocationModes = InvocationMode.parseArguments(invocationModes) |
| ..verbosity = verbosity; |
| } |
| |
| abstract class Compiler { |
| final int isolateGroupId; |
| final FileSystem fileSystem; |
| final Uri? platformKernelPath; |
| final bool enableAsserts; |
| final int nullSafety; |
| final List<String>? experimentalFlags; |
| final String? packageConfig; |
| final String invocationModes; |
| final String verbosityLevel; |
| |
| // Code coverage and hot reload are only supported by incremental compiler, |
| // which is used if vm-service is enabled. |
| final bool supportCodeCoverage; |
| final bool supportHotReload; |
| |
| final List<String> errorsPlain = <String>[]; |
| final List<String> errorsColorized = <String>[]; |
| |
| late final CompilerOptions options; |
| |
| Compiler(this.isolateGroupId, this.fileSystem, this.platformKernelPath, |
| {this.enableAsserts: false, |
| this.nullSafety: kNullSafetyOptionUnspecified, |
| this.experimentalFlags: null, |
| this.supportCodeCoverage: false, |
| this.supportHotReload: false, |
| this.packageConfig: null, |
| this.invocationModes: '', |
| this.verbosityLevel: Verbosity.defaultValue}) { |
| Uri? packagesUri = null; |
| final packageConfig = this.packageConfig ?? Platform.packageConfig; |
| if (packageConfig != null) { |
| packagesUri = Uri.parse(packageConfig); |
| } |
| |
| if (verbose) { |
| print("DFE: Platform.packageConfig: ${Platform.packageConfig}"); |
| print("DFE: packagesUri: ${packagesUri}"); |
| print("DFE: Platform.resolvedExecutable: ${Platform.resolvedExecutable}"); |
| print("DFE: platformKernelPath: ${platformKernelPath}"); |
| } |
| |
| options = setupCompilerOptions( |
| fileSystem, |
| platformKernelPath, |
| enableAsserts, |
| nullSafety, |
| experimentalFlags, |
| packagesUri, |
| errorsPlain, |
| errorsColorized, |
| invocationModes, |
| verbosityLevel); |
| } |
| |
| Future<CompilerResult> compile(Uri script) { |
| return runWithPrintToStderr(() async { |
| final CompilerResult compilerResult = await compileInternal(script); |
| final Component? component = compilerResult.component; |
| |
| if (errorsPlain.isEmpty) { |
| // Record dependencies only if compilation was error free. |
| _recordDependencies(isolateGroupId, component, options.packagesFileUri); |
| } |
| |
| return compilerResult; |
| }); |
| } |
| |
| Future<CompilerResult> compileInternal(Uri script); |
| } |
| |
| class CompilerResult { |
| final Component? component; |
| |
| /// Set of libraries loaded from .dill, with or without the SDK depending on |
| /// the compilation settings. |
| final Set<Library> loadedLibraries; |
| final ClassHierarchy? classHierarchy; |
| final CoreTypes? coreTypes; |
| |
| CompilerResult(this.component, this.loadedLibraries, this.classHierarchy, |
| this.coreTypes); |
| } |
| |
| // Environment map which looks up environment defines in the VM environment |
| // at runtime. |
| // TODO(askesc): This is a temporary hack to get hold of the environment during |
| // JIT compilation. We use a lazy map accessing the VM runtime environment using |
| // new String.fromEnvironment, since the VM currently does not support providing |
| // the full (isolate specific) environment as a finite, static map. |
| class EnvironmentMap extends UnmodifiableMapBase<String, String> { |
| @override |
| bool containsKey(Object? key) { |
| return key is String && new bool.hasEnvironment(key); |
| } |
| |
| @override |
| String? operator [](Object? key) { |
| // The fromEnvironment constructor is specified to throw when called using |
| // new. However, the VM implementation actually looks up the given name in |
| // the environment. |
| if (containsKey(key)) { |
| return new String.fromEnvironment(key as String); |
| } |
| return null; |
| } |
| |
| @override |
| get keys => throw "Environment map iteration not supported"; |
| } |
| |
| class FileSink implements Sink<List<int>> { |
| MemoryFileSystemEntity entityForUri; |
| List<int> bytes = <int>[]; |
| |
| FileSink(this.entityForUri); |
| |
| @override |
| void add(List<int> data) { |
| bytes.addAll(data); |
| } |
| |
| @override |
| void close() { |
| this.entityForUri.writeAsBytesSync(bytes); |
| } |
| } |
| |
| class IncrementalCompilerWrapper extends Compiler { |
| IncrementalCompiler? generator; |
| |
| IncrementalCompilerWrapper( |
| int isolateGroupId, FileSystem fileSystem, Uri? platformKernelPath, |
| {bool enableAsserts: false, |
| int nullSafety: kNullSafetyOptionUnspecified, |
| List<String>? experimentalFlags, |
| String? packageConfig, |
| String invocationModes: '', |
| String verbosityLevel: Verbosity.defaultValue}) |
| : super(isolateGroupId, fileSystem, platformKernelPath, |
| enableAsserts: enableAsserts, |
| nullSafety: nullSafety, |
| experimentalFlags: experimentalFlags, |
| supportHotReload: true, |
| supportCodeCoverage: true, |
| packageConfig: packageConfig, |
| invocationModes: invocationModes, |
| verbosityLevel: verbosityLevel); |
| |
| factory IncrementalCompilerWrapper.forExpressionCompilationOnly( |
| Component component, |
| int isolateGroupId, |
| FileSystem fileSystem, |
| Uri? platformKernelPath, |
| {bool enableAsserts: false, |
| List<String>? experimentalFlags, |
| String? packageConfig, |
| String invocationModes: ''}) { |
| IncrementalCompilerWrapper result = IncrementalCompilerWrapper( |
| isolateGroupId, fileSystem, platformKernelPath, |
| enableAsserts: enableAsserts, |
| experimentalFlags: experimentalFlags, |
| packageConfig: packageConfig, |
| invocationModes: invocationModes); |
| result.generator = new IncrementalCompiler.forExpressionCompilationOnly( |
| component, |
| result.options, |
| component.mainMethod!.enclosingLibrary.fileUri); |
| return result; |
| } |
| |
| @override |
| Future<CompilerResult> compileInternal(Uri script) async { |
| final generator = this.generator ??= IncrementalCompiler(options, script); |
| errorsPlain.clear(); |
| errorsColorized.clear(); |
| final compilerResult = await generator.compile(entryPoint: script); |
| final component = compilerResult.component; |
| return new CompilerResult(component, const {}, |
| compilerResult.classHierarchy, compilerResult.coreTypes); |
| } |
| |
| void accept() => generator!.accept(); |
| void invalidate(Uri uri) => generator!.invalidate(uri); |
| |
| Future<IncrementalCompilerWrapper> clone(int isolateGroupId) async { |
| IncrementalCompilerWrapper clone = IncrementalCompilerWrapper( |
| isolateGroupId, fileSystem, platformKernelPath, |
| enableAsserts: enableAsserts, |
| nullSafety: nullSafety, |
| experimentalFlags: experimentalFlags, |
| packageConfig: packageConfig, |
| invocationModes: invocationModes); |
| final generator = this.generator!; |
| // TODO(VM TEAM): This does not seem safe. What if cloning while having |
| // pending deltas for instance? |
| generator.resetDeltaState(); |
| IncrementalCompilerResult compilerResult = await generator.compile(); |
| Component fullComponent = compilerResult.component; |
| |
| // Assume fileSystem is HybridFileSystem because that is the setup where |
| // clone should be used for. |
| MemoryFileSystem memoryFileSystem = (fileSystem as HybridFileSystem).memory; |
| |
| String filename = 'full-component-$isolateGroupId.dill'; |
| Sink<List<int>> sink = |
| FileSink(memoryFileSystem.entityForUri(Uri.file(filename))); |
| new BinaryPrinter(sink).writeComponentFile(fullComponent); |
| sink.close(); |
| |
| clone.generator = new IncrementalCompiler(options, generator.entryPoint, |
| initializeFromDillUri: Uri.file(filename)); |
| return clone; |
| } |
| } |
| |
| class SingleShotCompilerWrapper extends Compiler { |
| final bool requireMain; |
| |
| SingleShotCompilerWrapper( |
| int isolateGroupId, FileSystem fileSystem, Uri platformKernelPath, |
| {this.requireMain: false, |
| bool enableAsserts: false, |
| int nullSafety: kNullSafetyOptionUnspecified, |
| List<String>? experimentalFlags, |
| String? packageConfig, |
| String invocationModes: '', |
| String verbosityLevel: Verbosity.defaultValue}) |
| : super(isolateGroupId, fileSystem, platformKernelPath, |
| enableAsserts: enableAsserts, |
| nullSafety: nullSafety, |
| experimentalFlags: experimentalFlags, |
| packageConfig: packageConfig, |
| invocationModes: invocationModes, |
| verbosityLevel: verbosityLevel); |
| |
| @override |
| Future<CompilerResult> compileInternal(Uri script) async { |
| final fe.CompilerResult? compilerResult = requireMain |
| ? await kernelForProgram(script, options) |
| : await kernelForModule([script], options); |
| if (compilerResult == null) { |
| return CompilerResult(null, const {}, null, null); |
| } |
| |
| Set<Library> loadedLibraries = createLoadedLibrariesSet( |
| compilerResult.loadedComponents, compilerResult.sdkComponent, |
| includePlatform: !options.omitPlatform); |
| |
| return new CompilerResult(compilerResult.component, loadedLibraries, |
| compilerResult.classHierarchy, compilerResult.coreTypes); |
| } |
| } |
| |
| final Map<int, IncrementalCompilerWrapper> isolateCompilers = {}; |
| final Map<int, List<Uri>> isolateDependencies = {}; |
| final Map<int, _ExpressionCompilationFromDillSettings> isolateLoadNotifies = {}; |
| |
| IncrementalCompilerWrapper? lookupIncrementalCompiler(int isolateGroupId) { |
| return isolateCompilers[isolateGroupId]; |
| } |
| |
| Future<Compiler> lookupOrBuildNewIncrementalCompiler(int isolateGroupId, |
| List sourceFiles, Uri platformKernelPath, List<int>? platformKernel, |
| {bool enableAsserts: false, |
| int nullSafety: kNullSafetyOptionUnspecified, |
| List<String>? experimentalFlags, |
| String? packageConfig, |
| String? multirootFilepaths, |
| String? multirootScheme, |
| String invocationModes: '', |
| String verbosityLevel: Verbosity.defaultValue}) async { |
| IncrementalCompilerWrapper? compiler = |
| lookupIncrementalCompiler(isolateGroupId); |
| if (compiler != null) { |
| updateSources(compiler, sourceFiles); |
| invalidateSources(compiler, sourceFiles); |
| } else { |
| // This is how identify scenario where child isolate hot reload requests |
| // requires setting up actual compiler first: non-empty sourceFiles list has |
| // no actual content specified for the source file. |
| if (sourceFiles.isNotEmpty && sourceFiles[1] == null) { |
| // Just use first compiler that should represent main isolate as a source for cloning. |
| var source = isolateCompilers.entries.first; |
| compiler = await source.value.clone(isolateGroupId); |
| } else { |
| FileSystem fileSystem = _buildFileSystem( |
| sourceFiles, platformKernel, multirootFilepaths, multirootScheme); |
| |
| // TODO(aam): IncrementalCompilerWrapper instance created below have to be |
| // destroyed when corresponding isolate is shut down. To achieve that kernel |
| // isolate needs to receive a message indicating that particular |
| // isolate was shut down. Message should be handled here in this script. |
| compiler = new IncrementalCompilerWrapper( |
| isolateGroupId, fileSystem, platformKernelPath, |
| enableAsserts: enableAsserts, |
| nullSafety: nullSafety, |
| experimentalFlags: experimentalFlags, |
| packageConfig: packageConfig, |
| invocationModes: invocationModes, |
| verbosityLevel: verbosityLevel); |
| } |
| isolateCompilers[isolateGroupId] = compiler; |
| } |
| return compiler; |
| } |
| |
| void updateSources(IncrementalCompilerWrapper compiler, List sourceFiles) { |
| final bool hasMemoryFS = compiler.fileSystem is HybridFileSystem; |
| if (sourceFiles.isNotEmpty) { |
| final FileSystem fs = compiler.fileSystem; |
| for (int i = 0; i < sourceFiles.length ~/ 2; i++) { |
| Uri uri = Uri.parse(sourceFiles[i * 2]); |
| List<int>? source = sourceFiles[i * 2 + 1]; |
| // The source is only provided by unit tests and is normally empty. |
| // Don't add an entry for the uri so the compiler will fallback to the |
| // real file system for the updated source. |
| if (hasMemoryFS && source != null) { |
| (fs as HybridFileSystem) |
| .memory |
| .entityForUri(uri) |
| .writeAsBytesSync(source); |
| } |
| } |
| } |
| } |
| |
| void invalidateSources(IncrementalCompilerWrapper compiler, List sourceFiles) { |
| if (sourceFiles.isNotEmpty) { |
| for (int i = 0; i < sourceFiles.length ~/ 2; i++) { |
| compiler.invalidate(Uri.parse(sourceFiles[i * 2])); |
| } |
| } |
| } |
| |
| // Process a request from the runtime. See KernelIsolate::CompileToKernel in |
| // kernel_isolate.cc and Loader::SendKernelRequest in loader.cc. |
| Future _processExpressionCompilationRequest(request) async { |
| final SendPort port = request[1]; |
| final int isolateGroupId = request[2]; |
| final dynamic dart_platform_kernel = request[3]; |
| final String expression = request[4]; |
| final List<String> definitions = request[5].cast<String>(); |
| final List<String> typeDefinitions = request[6].cast<String>(); |
| final String libraryUri = request[7]; |
| final String? klass = request[8]; |
| final String? method = request[9]; |
| final bool isStatic = request[10]; |
| final List<List<int>> dillData = request[11].cast<List<int>>(); |
| final int blobLoadCount = request[12]; |
| final bool enableAsserts = request[13]; |
| final List<String>? experimentalFlags = |
| request[14] != null ? request[14].cast<String>() : null; |
| |
| IncrementalCompilerWrapper? compiler = isolateCompilers[isolateGroupId]; |
| |
| _ExpressionCompilationFromDillSettings? isolateLoadDillData = |
| isolateLoadNotifies[isolateGroupId]; |
| if (isolateLoadDillData != null) { |
| // Check if we can reuse the compiler. |
| if (isolateLoadDillData.blobLoadCount != blobLoadCount || |
| isolateLoadDillData.prevDillCount != dillData.length) { |
| isolateCompilers.remove(isolateGroupId); |
| compiler = null; |
| } |
| } |
| |
| if (compiler == null) { |
| if (dillData.isNotEmpty) { |
| if (verbose) { |
| print("DFE: Initializing compiler from ${dillData.length} dill files"); |
| } |
| isolateLoadNotifies[isolateGroupId] = |
| new _ExpressionCompilationFromDillSettings( |
| blobLoadCount, dillData.length); |
| |
| // Create Component initialized from the bytes. |
| Component component = new Component(); |
| |
| // First try to just load all "dillData". This *might* include the |
| // platform (and we might have the (same) platform both here and in |
| // dart_platform_kernel). |
| for (List<int> bytes in dillData) { |
| // TODO(jensj): There might be an issue if main has changed. |
| new BinaryBuilderWithMetadata(bytes, alwaysCreateNewNamedNodes: true) |
| .readComponent(component); |
| } |
| |
| // Check if the loaded component has the platform. |
| // If it does not, try to load from dart_platform_kernel or from file. |
| bool foundDartCore = false; |
| for (Library library in component.libraries) { |
| if (library.importUri.scheme == "dart" && |
| library.importUri.path == "core" && |
| !library.isSynthetic) { |
| foundDartCore = true; |
| break; |
| } |
| } |
| if (!foundDartCore) { |
| List<int> platformKernel; |
| if (dart_platform_kernel is List<int>) { |
| platformKernel = dart_platform_kernel; |
| } else { |
| final Uri platformUri = computePlatformBinariesLocation() |
| .resolve('vm_platform_strong.dill'); |
| final File platformFile = new File.fromUri(platformUri); |
| if (platformFile.existsSync()) { |
| platformKernel = platformFile.readAsBytesSync(); |
| } else { |
| port.send(new CompilationResult.errors( |
| ["No platform found to initialize incremental compiler."], |
| null) |
| .toResponse()); |
| return; |
| } |
| } |
| |
| new BinaryBuilderWithMetadata(platformKernel, |
| alwaysCreateNewNamedNodes: true) |
| .readComponent(component); |
| } |
| |
| FileSystem fileSystem = |
| _buildFileSystem([dotPackagesFile, <int>[]], null, null, null); |
| |
| // TODO(aam): IncrementalCompilerWrapper instance created below have to be |
| // destroyed when corresponding isolate is shut down. To achieve that |
| // kernel isolate needs to receive a message indicating that particular |
| // isolate was shut down. Message should be handled here in this script. |
| try { |
| compiler = new IncrementalCompilerWrapper.forExpressionCompilationOnly( |
| component, isolateGroupId, fileSystem, null, |
| enableAsserts: enableAsserts, |
| experimentalFlags: experimentalFlags, |
| packageConfig: dotPackagesFile); |
| isolateCompilers[isolateGroupId] = compiler; |
| await compiler.compile( |
| component.mainMethod?.enclosingLibrary.importUri ?? |
| component.libraries.last.importUri); |
| } catch (e) { |
| port.send(new CompilationResult.errors([ |
| "Error when trying to create a compiler for expression compilation: " |
| "'$e'." |
| ], null) |
| .toResponse()); |
| return; |
| } |
| } |
| } |
| |
| if (compiler == null) { |
| port.send(new CompilationResult.errors( |
| ["No incremental compiler available for this isolate."], null) |
| .toResponse()); |
| return; |
| } |
| |
| compiler.errorsPlain.clear(); |
| compiler.errorsColorized.clear(); |
| |
| CompilationResult result; |
| try { |
| Procedure? procedure = await compiler.generator!.compileExpression( |
| expression, |
| definitions, |
| typeDefinitions, |
| libraryUri, |
| klass, |
| method, |
| isStatic); |
| |
| if (procedure == null) { |
| port.send( |
| new CompilationResult.errors(["Invalid scope."], null).toResponse()); |
| return; |
| } |
| |
| assert(compiler.errorsPlain.length == compiler.errorsColorized.length); |
| // Any error will be printed verbatim in observatory, so we always use the |
| // plain version (i.e. the one without ANSI escape codes in it). |
| if (compiler.errorsPlain.isNotEmpty) { |
| // TODO(sigmund): the compiler prints errors to the console, so we |
| // shouldn't print those messages again here. |
| result = new CompilationResult.errors(compiler.errorsPlain, null); |
| } else { |
| Component component = createExpressionEvaluationComponent(procedure); |
| result = new CompilationResult.ok(serializeComponent(component)); |
| } |
| } catch (error, stack) { |
| result = new CompilationResult.crash(error, stack); |
| } |
| |
| port.send(result.toResponse()); |
| } |
| |
| void _recordDependencies( |
| int isolateGroupId, Component? component, Uri? packageConfig) { |
| final dependencies = isolateDependencies[isolateGroupId] ??= <Uri>[]; |
| |
| if (component != null) { |
| for (var lib in component.libraries) { |
| if (lib.importUri.scheme == "dart") continue; |
| |
| dependencies.add(lib.fileUri); |
| for (var part in lib.parts) { |
| final fileUri = lib.fileUri.resolve(part.partUri); |
| if (fileUri.scheme != "" && fileUri.scheme != "file") { |
| // E.g. part 'package:foo/foo.dart'; |
| // Maybe the front end should resolve this? |
| continue; |
| } |
| dependencies.add(fileUri); |
| } |
| } |
| } |
| |
| if (packageConfig != null) { |
| dependencies.add(packageConfig); |
| } |
| } |
| |
| String _escapeDependency(Uri uri) { |
| return uri.toFilePath().replaceAll("\\", "\\\\").replaceAll(" ", "\\ "); |
| } |
| |
| Uint8List _serializeDependencies(List<Uri> uris) { |
| return utf8.encode(uris.map(_escapeDependency).join(" ")) as Uint8List; |
| } |
| |
| Future _processListDependenciesRequest( |
| SendPort port, int isolateGroupId) async { |
| final List<Uri> dependencies = isolateDependencies[isolateGroupId] ?? <Uri>[]; |
| |
| CompilationResult result; |
| try { |
| result = new CompilationResult.ok(_serializeDependencies(dependencies)); |
| } catch (error, stack) { |
| result = new CompilationResult.crash(error, stack); |
| } |
| |
| port.send(result.toResponse()); |
| } |
| |
| Future _processIsolateShutdownNotification(request) async { |
| final int isolateGroupId = request[1]; |
| isolateCompilers.remove(isolateGroupId); |
| isolateDependencies.remove(isolateGroupId); |
| isolateLoadNotifies.remove(isolateGroupId); |
| } |
| |
| Future _processLoadRequest(request) async { |
| if (verbose) { |
| for (int i = 0; i < request.length; i++) { |
| var part = request[i]; |
| String partToString; |
| if (part is List && part.isNotEmpty) { |
| // Assume this is large and printing all of it takes a lot of time. |
| StringBuffer sb = new StringBuffer(); |
| String prepend = "["; |
| for (int j = 0; j < part.length; j++) { |
| sb.write(prepend); |
| sb.write(part[j]); |
| prepend = ", "; |
| if (sb.length > 256) break; |
| } |
| sb.write("]"); |
| partToString = sb.toString(); |
| } else { |
| partToString = part.toString(); |
| } |
| if (partToString.length > 256) { |
| partToString = partToString.substring(0, 255) + "..."; |
| } |
| print("DFE: request[$i]: $partToString"); |
| } |
| } |
| |
| int tag = request[0]; |
| |
| if (tag == kCompileExpressionTag) { |
| await _processExpressionCompilationRequest(request); |
| return; |
| } |
| |
| if (tag == kNotifyIsolateShutdownTag) { |
| await _processIsolateShutdownNotification(request); |
| return; |
| } |
| |
| final SendPort port = request[1]; |
| final int isolateGroupId = request[7]; |
| |
| if (tag == kListDependenciesTag) { |
| await _processListDependenciesRequest(port, isolateGroupId); |
| return; |
| } |
| |
| final String? inputFileUri = request[2]; |
| final Uri? script = |
| inputFileUri != null ? Uri.base.resolve(inputFileUri) : null; |
| final bool incremental = request[4]; |
| final bool snapshot = request[5]; |
| final int nullSafety = request[6]; |
| final List sourceFiles = request[8]; |
| final bool enableAsserts = request[9]; |
| final List<String>? experimentalFlags = |
| request[10] != null ? request[10].cast<String>() : null; |
| final String? packageConfig = request[11]; |
| final String? multirootFilepaths = request[12]; |
| final String? multirootScheme = request[13]; |
| final String? workingDirectory = request[14]; |
| final String verbosityLevel = request[15]; |
| Uri platformKernelPath; |
| List<int>? platformKernel = null; |
| if (request[3] is String) { |
| platformKernelPath = Uri.base.resolveUri(new Uri.file(request[3])); |
| } else if (request[3] is List<int>) { |
| platformKernelPath = Uri.parse(platformKernelFile); |
| platformKernel = request[3]; |
| } else { |
| platformKernelPath = |
| computePlatformBinariesLocation().resolve('vm_platform_strong.dill'); |
| } |
| |
| final String invocationModes = snapshot ? 'compile' : ''; |
| |
| Compiler? compiler; |
| |
| // Update the in-memory file system with the provided sources. Currently, only |
| // unit tests compile sources that are not on the file system, so this can only |
| // happen during unit tests. |
| if (tag == kUpdateSourcesTag) { |
| assert(incremental, |
| "Incremental compiler required for use of 'kUpdateSourcesTag'"); |
| compiler = lookupIncrementalCompiler(isolateGroupId); |
| if (compiler == null) { |
| port.send(new CompilationResult.errors( |
| ["No incremental compiler available for this isolate."], null) |
| .toResponse()); |
| return; |
| } |
| updateSources(compiler as IncrementalCompilerWrapper, sourceFiles); |
| port.send(new CompilationResult.ok(null).toResponse()); |
| return; |
| } else if (tag == kAcceptTag) { |
| assert( |
| incremental, "Incremental compiler required for use of 'kAcceptTag'"); |
| compiler = lookupIncrementalCompiler(isolateGroupId); |
| // There are unit tests that invoke the IncrementalCompiler directly and |
| // request a reload, meaning that we won't have a compiler for this isolate. |
| if (compiler != null) { |
| (compiler as IncrementalCompilerWrapper).accept(); |
| } |
| port.send(new CompilationResult.ok(null).toResponse()); |
| return; |
| } else if (tag == kDetectNullabilityTag) { |
| FileSystem fileSystem = _buildFileSystem( |
| sourceFiles, platformKernel, multirootFilepaths, multirootScheme); |
| Uri? packagesUri = null; |
| final packageConfigWithDefault = packageConfig ?? Platform.packageConfig; |
| if (packageConfigWithDefault != null) { |
| packagesUri = Uri.parse(packageConfigWithDefault); |
| } |
| if (packagesUri != null && packagesUri.scheme == '') { |
| // Script does not have a scheme, assume that it is a path, |
| // resolve it against the working directory. |
| packagesUri = Uri.directory(workingDirectory!).resolveUri(packagesUri); |
| } |
| final List<String> errorsPlain = <String>[]; |
| final List<String> errorsColorized = <String>[]; |
| var options = setupCompilerOptions( |
| fileSystem, |
| platformKernelPath, |
| false, |
| nullSafety, |
| experimentalFlags, |
| packagesUri, |
| errorsPlain, |
| errorsColorized, |
| invocationModes, |
| verbosityLevel); |
| |
| // script should only be null for kUpdateSourcesTag. |
| await autoDetectNullSafetyMode(script!, options); |
| bool value = options.nnbdMode == NnbdMode.Strong; |
| port.send(new CompilationResult.nullSafety(value).toResponse()); |
| return; |
| } |
| |
| // script should only be null for kUpdateSourcesTag. |
| assert(script != null); |
| |
| // TODO(aam): There should be no need to have an option to choose |
| // one compiler or another. We should always use an incremental |
| // compiler as its functionality is a super set of the other one. We need to |
| // watch the performance though. |
| if (incremental) { |
| compiler = await lookupOrBuildNewIncrementalCompiler( |
| isolateGroupId, sourceFiles, platformKernelPath, platformKernel, |
| enableAsserts: enableAsserts, |
| nullSafety: nullSafety, |
| experimentalFlags: experimentalFlags, |
| packageConfig: packageConfig, |
| multirootFilepaths: multirootFilepaths, |
| multirootScheme: multirootScheme, |
| invocationModes: invocationModes, |
| verbosityLevel: verbosityLevel); |
| } else { |
| FileSystem fileSystem = _buildFileSystem( |
| sourceFiles, platformKernel, multirootFilepaths, multirootScheme); |
| compiler = new SingleShotCompilerWrapper( |
| isolateGroupId, fileSystem, platformKernelPath, |
| requireMain: false, |
| enableAsserts: enableAsserts, |
| nullSafety: nullSafety, |
| experimentalFlags: experimentalFlags, |
| packageConfig: packageConfig, |
| invocationModes: invocationModes, |
| verbosityLevel: verbosityLevel); |
| } |
| |
| CompilationResult result; |
| try { |
| if (verbose) { |
| print("DFE: scriptUri: ${script}"); |
| } |
| |
| CompilerResult compilerResult = await compiler.compile(script!); |
| Set<Library> loadedLibraries = compilerResult.loadedLibraries; |
| |
| assert(compiler.errorsPlain.length == compiler.errorsColorized.length); |
| // http://dartbug.com/45137 |
| // enableColors calls `stdout.supportsAnsiEscapes` which - on Windows - |
| // does something with line endings. To avoid this when no error |
| // messages are do be printed anyway, we are carefull not to call it unless |
| // neccessary. |
| if (compiler.errorsColorized.isNotEmpty) { |
| final List<String> errors = |
| (enableColors) ? compiler.errorsColorized : compiler.errorsPlain; |
| final component = compilerResult.component; |
| if (component != null) { |
| result = new CompilationResult.errors( |
| errors, |
| serializeComponent(component, |
| filter: (lib) => !loadedLibraries.contains(lib))); |
| } else { |
| result = new CompilationResult.errors(errors, null); |
| } |
| } else { |
| // We serialize the component excluding vm_platform.dill because the VM has |
| // these sources built-in. Everything loaded as a summary in |
| // [kernelForProgram] is marked `external`, so we can use that bit to |
| // decide what to exclude. |
| result = new CompilationResult.ok(serializeComponent( |
| compilerResult.component!, |
| filter: (lib) => !loadedLibraries.contains(lib))); |
| } |
| } catch (error, stack) { |
| result = new CompilationResult.crash(error, stack); |
| } |
| |
| if (verbose) print("DFE:> ${result}"); |
| |
| if (tag == kTrainTag) { |
| // In training mode make sure to read the sdk a few more times... |
| ProcessedOptions p = new ProcessedOptions(options: compiler.options); |
| final bytes = (await p.loadSdkSummaryBytes())!; |
| for (int i = 0; i < 5; i++) { |
| p.loadComponent(bytes, null); |
| } |
| |
| if (result.status != Status.ok) { |
| tag = -tag; |
| } |
| port.send([tag, inputFileUri, inputFileUri, null, result.payload]); |
| } else if (tag == kCompileTag) { |
| port.send(result.toResponse()); |
| } else { |
| port.send([ |
| -tag, |
| inputFileUri, |
| inputFileUri, |
| null, |
| new CompilationResult.errors(<String>["unknown tag"], null).payload |
| ]); |
| } |
| } |
| |
| /// Creates a file system containing the files specified in [sourceFiles] and |
| /// that delegates to the underlying file system for any other file request. |
| /// The [sourceFiles] list interleaves file name string and |
| /// raw file content Uint8List. |
| /// |
| /// The result can be used instead of StandardFileSystem.instance by the |
| /// frontend. |
| FileSystem _buildFileSystem(List sourceFiles, List<int>? platformKernel, |
| String? multirootFilepaths, String? multirootScheme) { |
| FileSystem fileSystem = new HttpAwareFileSystem(StandardFileSystem.instance); |
| |
| if (!sourceFiles.isEmpty || platformKernel != null) { |
| MemoryFileSystem memoryFileSystem = |
| new MemoryFileSystem(Uri.parse('file:///')); |
| for (int i = 0; i < sourceFiles.length ~/ 2; i++) { |
| memoryFileSystem |
| .entityForUri(Uri.parse(sourceFiles[i * 2])) |
| .writeAsBytesSync(sourceFiles[i * 2 + 1]); |
| } |
| if (platformKernel != null) { |
| memoryFileSystem |
| .entityForUri(Uri.parse(platformKernelFile)) |
| .writeAsBytesSync(platformKernel); |
| } |
| fileSystem = new HybridFileSystem(memoryFileSystem, fileSystem); |
| } |
| |
| if (multirootFilepaths != null) { |
| List<Uri> list = multirootFilepaths |
| .split(',') |
| .map((String s) => Uri.base.resolveUri(new Uri.file(s))) |
| .toList(); |
| fileSystem = new MultiRootFileSystem( |
| multirootScheme ?? "org-dartlang-root", list, fileSystem); |
| } |
| return fileSystem; |
| } |
| |
| train(String scriptUri, String? platformKernelPath) async { |
| // Train on program asked to train on. |
| await trainInternal(scriptUri, platformKernelPath); |
| |
| // Also train a few times on a hello-world program to make sure we exercise |
| // the startup sequence. |
| Directory tmpDir = |
| Directory.systemTemp.createTempSync("kernel_service_train"); |
| File helloDart = new File.fromUri(tmpDir.uri.resolve("hello.dart")); |
| helloDart.writeAsStringSync(""" |
| main() { |
| print("Hello, World!"); |
| } |
| """); |
| try { |
| for (int i = 0; i < 10; i++) { |
| await trainInternal(helloDart.uri.toString(), platformKernelPath); |
| } |
| } finally { |
| tmpDir.deleteSync(recursive: true); |
| } |
| } |
| |
| Future trainInternal(String scriptUri, String? platformKernelPath) async { |
| var tag = kTrainTag; |
| var responsePort = new RawReceivePort(); |
| responsePort.handler = (response) { |
| if (response[0] == tag) { |
| // Success. |
| responsePort.close(); |
| } else if (response[0] == -tag) { |
| // Compilation error. |
| throw response[4]; |
| } else { |
| throw "Unexpected response: $response"; |
| } |
| }; |
| var request = [ |
| tag, |
| responsePort.sendPort, |
| scriptUri, |
| platformKernelPath, |
| false /* incremental */, |
| false /* snapshot */, |
| kNullSafetyOptionUnspecified /* null safety */, |
| 1 /* isolateGroupId chosen randomly */, |
| [] /* source files */, |
| false /* enable asserts */, |
| null /* experimental_flags */, |
| null /* package_config */, |
| null /* multirootFilepaths */, |
| null /* multirootScheme */, |
| null /* original working directory */, |
| 'all' /* CFE logging mode */, |
| ]; |
| await _processLoadRequest(request); |
| } |
| |
| main([args]) { |
| if ((args?.length ?? 0) > 1 && args[0] == '--train') { |
| // This entry point is used when creating an app snapshot. |
| // It takes the following extra arguments: |
| // 1) Script to compile. |
| // 2) Optional platform kernel path. |
| int argIndex = 1; |
| final String script = args[argIndex++]; |
| final String? platform = (argIndex < args.length) ? args[argIndex] : null; |
| train(script, platform); |
| } else { |
| // Entry point for the Kernel isolate. |
| return new RawReceivePort()..handler = _processLoadRequest; |
| } |
| } |
| |
| /// Compilation status codes. |
| /// |
| /// Note: The [index] property of these constants must match |
| /// `Dart_KernelCompilationStatus` in |
| /// [dart_api.h](../../../../runtime/include/dart_api.h). |
| enum Status { |
| /// Compilation was successful. |
| ok, |
| |
| /// Compilation failed with a compile time error. |
| error, |
| |
| /// Compiler crashed. |
| crash, |
| } |
| |
| abstract class CompilationResult { |
| CompilationResult._(); |
| |
| factory CompilationResult.ok(Uint8List? bytes) = _CompilationOk; |
| |
| factory CompilationResult.nullSafety(bool val) = _CompilationNullSafety; |
| |
| factory CompilationResult.errors(List<String> errors, Uint8List? bytes) = |
| _CompilationError; |
| |
| factory CompilationResult.crash(Object exception, StackTrace stack) = |
| _CompilationCrash; |
| |
| Status get status; |
| |
| get payload; |
| |
| List toResponse() => [status.index, payload]; |
| } |
| |
| class _CompilationOk extends CompilationResult { |
| final Uint8List? bytes; |
| |
| _CompilationOk(this.bytes) : super._() { |
| if (dumpKernel) { |
| final bytes = this.bytes; |
| if (bytes != null) { |
| _debugDumpKernel(bytes); |
| } |
| } |
| } |
| |
| @override |
| Status get status => Status.ok; |
| |
| @override |
| get payload => bytes; |
| |
| String toString() => "_CompilationOk(${bytes?.length ?? 0} bytes)"; |
| } |
| |
| class _CompilationNullSafety extends CompilationResult { |
| final bool _null_safety; |
| |
| _CompilationNullSafety(this._null_safety) : super._() {} |
| |
| @override |
| Status get status => Status.ok; |
| |
| @override |
| get payload => _null_safety; |
| |
| String toString() => "_CompilationNullSafety($_null_safety)"; |
| } |
| |
| abstract class _CompilationFail extends CompilationResult { |
| _CompilationFail() : super._(); |
| |
| String get errorString; |
| |
| @override |
| get payload => errorString; |
| } |
| |
| class _CompilationError extends _CompilationFail { |
| final Uint8List? bytes; |
| final List<String> errors; |
| |
| _CompilationError(this.errors, this.bytes); |
| |
| @override |
| Status get status => Status.error; |
| |
| @override |
| String get errorString => errors.take(10).join('\n'); |
| |
| String toString() => "_CompilationError(${errorString})"; |
| |
| List toResponse() => [status.index, payload, bytes]; |
| } |
| |
| class _CompilationCrash extends _CompilationFail { |
| final Object exception; |
| final StackTrace stack; |
| |
| _CompilationCrash(this.exception, this.stack); |
| |
| @override |
| Status get status => Status.crash; |
| |
| @override |
| String get errorString => "${exception}\n${stack}"; |
| |
| String toString() => "_CompilationCrash(${errorString})"; |
| } |
| |
| Future<T> runWithPrintToStderr<T>(Future<T> f()) { |
| return runZoned(() => new Future<T>(f), |
| zoneSpecification: new ZoneSpecification( |
| print: (_1, _2, _3, String line) => stderr.writeln(line))); |
| } |
| |
| int _debugDumpCounter = 0; |
| void _debugDumpKernel(Uint8List bytes) { |
| new File('kernel_service.tmp${_debugDumpCounter++}.dill') |
| .writeAsBytesSync(bytes); |
| } |
| |
| class _ExpressionCompilationFromDillSettings { |
| int blobLoadCount; |
| int prevDillCount; |
| |
| _ExpressionCompilationFromDillSettings( |
| this.blobLoadCount, this.prevDillCount); |
| } |