blob: a374fc27fddb94f2d5976586e95e5a892d6c1a6e [file] [log] [blame]
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
/// A tool that invokes the CFE to compute kernel summary files.
/// This script can be used as a command-line command or a persistent server.
/// The server is implemented using the bazel worker protocol, so it can be used
/// within bazel as is. Other tools (like pub-build and package-build) also
/// use this persistent worker via the same protocol.
import 'dart:async';
import 'dart:io';
import 'dart:isolate';
import 'package:args/args.dart';
import 'package:bazel_worker/bazel_worker.dart';
import 'package:build_integration/file_system/multi_root.dart';
import 'package:dev_compiler/src/kernel/target.dart';
import 'package:front_end/src/api_unstable/bazel_worker.dart' as fe;
import 'package:kernel/ast.dart' show Component, Library, Reference;
import 'package:kernel/target/targets.dart';
import 'package:vm/target/vm.dart';
import 'package:vm/target/flutter.dart';
import 'package:vm/target/flutter_runner.dart';
import 'package:compiler/src/kernel/dart2js_target.dart';
/// [sendPort] may be passed in when started in an isolate. If provided, it is
/// used for bazel worker communication instead of stdin/stdout.
main(List<String> args, SendPort sendPort) async {
args = preprocessArgs(args);
if (args.contains('--persistent_worker')) {
if (args.length != 1) {
throw new StateError(
"unexpected args, expected only --persistent-worker but got: $args");
await new KernelWorker(sendPort: sendPort).run();
} else {
var result = await computeKernel(args);
if (!result.succeeded) {
exitCode = 15;
/// A bazel worker loop that can compute full or summary kernel files.
class KernelWorker extends AsyncWorkerLoop {
fe.InitializedCompilerState previousState;
/// If [sendPort] is provided it is used for bazel worker communication
/// instead of stdin/stdout.
KernelWorker({SendPort sendPort})
: super(
connection: sendPort == null
? null
: SendPortAsyncWorkerConnection(sendPort));
Future<WorkResponse> performRequest(WorkRequest request) async {
var outputBuffer = new StringBuffer();
var response = new WorkResponse()..exitCode = 0;
try {
fe.InitializedCompilerState previousStateToPass;
if (request.arguments.contains("--reuse-compiler-result")) {
previousStateToPass = previousState;
} else {
previousState = null;
var result = await computeKernel(request.arguments,
isWorker: true,
outputBuffer: outputBuffer,
inputs: request.inputs,
previousState: previousStateToPass);
previousState = result.previousState;
if (!result.succeeded) {
response.exitCode = 15;
} catch (e, s) {
response.exitCode = 15;
response.output = outputBuffer.toString();
return response;
/// If the last arg starts with `@`, this reads the file it points to and treats
/// each line as an additional arg.
/// This is how individual work request args are differentiated from startup
/// args in bazel (inidividual work request args go in that file).
List<String> preprocessArgs(List<String> args) {
args = new List.from(args);
if (args.isEmpty) {
return args;
String lastArg = args.last;
if (lastArg.startsWith('@')) {
File argsFile = new File(lastArg.substring(1));
try {
} on FileSystemException catch (e) {
throw new Exception('Failed to read file specified by $lastArg : $e');
return args;
/// An [ArgParser] for generating kernel summaries.
final summaryArgsParser = new ArgParser()
..addFlag('help', negatable: false, abbr: 'h')
negatable: false,
help: 'Whether source files loaded implicitly should be included as '
'part of the summary.')
defaultsTo: true,
negatable: true,
help: 'Whether to only build summary files.')
allowed: const [
help: 'Build kernel for the vm, flutter, flutter_runner, dart2js or ddc')
..addOption('multi-root-scheme', defaultsTo: 'org-dartlang-multi-root')
..addFlag('reuse-compiler-result', defaultsTo: false)
..addFlag('use-incremental-compiler', defaultsTo: false)
..addFlag('track-widget-creation', defaultsTo: false)
help: 'Enable a language experiment when invoking the CFE.')
..addMultiOption('define', abbr: 'D');
class ComputeKernelResult {
final bool succeeded;
final fe.InitializedCompilerState previousState;
ComputeKernelResult(this.succeeded, this.previousState);
/// Computes a kernel file based on [args].
/// If [isWorker] is true then exit codes will not be set on failure.
/// If [outputBuffer] is provided then messages will be written to that buffer
/// instead of printed to the console.
/// Returns whether or not the summary was successfully output.
Future<ComputeKernelResult> computeKernel(List<String> args,
{bool isWorker: false,
StringBuffer outputBuffer,
Iterable<Input> inputs,
fe.InitializedCompilerState previousState}) async {
dynamic out = outputBuffer ?? stderr;
bool succeeded = true;
var parsedArgs = summaryArgsParser.parse(args);
if (parsedArgs['help']) {
if (!isWorker) exit(0);
return new ComputeKernelResult(false, previousState);
// Bazel creates an overlay file system where some files may be located in the
// source tree, some in a gendir, and some in a bindir. The multi-root file
// system hides this from the front end.
var multiRoots = parsedArgs['multi-root'].map(Uri.base.resolve).toList();
if (multiRoots.isEmpty) multiRoots.add(Uri.base);
var fileSystem = new MultiRootFileSystem(parsedArgs['multi-root-scheme'],
multiRoots, fe.StandardFileSystem.instance);
var sources = (parsedArgs['source'] as List<String>).map(_toUri).toList();
var excludeNonSources = parsedArgs['exclude-non-sources'] as bool;
var summaryOnly = parsedArgs['summary-only'] as bool;
var trackWidgetCreation = parsedArgs['track-widget-creation'] as bool;
// TODO(sigmund,jakemac): make target mandatory. We allow null to be backwards
// compatible while we migrate existing clients of this tool.
var targetName =
(parsedArgs['target'] as String) ?? (summaryOnly ? 'ddc' : 'vm');
var targetFlags = new TargetFlags(trackWidgetCreation: trackWidgetCreation);
Target target;
switch (targetName) {
case 'vm':
target = new VmTarget(targetFlags);
if (summaryOnly) {
out.writeln('error: --summary-only not supported for the vm target');
case 'flutter':
target = new FlutterTarget(targetFlags);
if (summaryOnly) {
throw new ArgumentError(
'error: --summary-only not supported for the flutter target');
case 'flutter_runner':
target = new FlutterRunnerTarget(targetFlags);
if (summaryOnly) {
throw new ArgumentError('error: --summary-only not supported for the '
'flutter_runner target');
case 'dart2js':
target = new Dart2jsTarget('dart2js', targetFlags);
if (summaryOnly) {
'error: --summary-only not supported for the dart2js target');
case 'ddc':
// TODO(jakemac):If `generateKernel` changes to return a summary
// component, process the component instead.
target =
new DevCompilerSummaryTarget(sources, excludeNonSources, targetFlags);
if (!summaryOnly) {
out.writeln('error: --no-summary-only not supported for the '
'ddc target');
out.writeln('error: unsupported target: $targetName');
List<Uri> linkedInputs =
(parsedArgs['input-linked'] as List<String>).map(_toUri).toList();
List<Uri> summaryInputs =
(parsedArgs['input-summary'] as List<String>).map(_toUri).toList();
fe.InitializedCompilerState state;
bool usingIncrementalCompiler = false;
bool recordUsedInputs = parsedArgs["used-inputs"] != null;
var environmentDefines = _parseEnvironmentDefines(parsedArgs['define']);
if (parsedArgs['use-incremental-compiler']) {
usingIncrementalCompiler = true;
/// Build a map of uris to digests.
final inputDigests = <Uri, List<int>>{};
if (inputs != null) {
for (var input in inputs) {
inputDigests[_toUri(input.path)] = input.digest;
// If digests weren't given and if not in worker mode, create fake data and
// ensure we don't have a previous state (as that wouldn't be safe with
// fake input digests).
if (!isWorker && inputDigests.isEmpty) {
previousState = null;
inputDigests[_toUri(parsedArgs['dart-sdk-summary'])] = const [0];
for (Uri uri in summaryInputs) {
inputDigests[uri] = const [0];
for (Uri uri in linkedInputs) {
inputDigests[uri] = const [0];
state = await fe.initializeIncrementalCompiler(
[...summaryInputs, ...linkedInputs],
(parsedArgs['enable-experiment'] as List<String>),
trackNeededDillLibraries: recordUsedInputs);
} else {
state = await fe.initializeCompiler(
// TODO(sigmund): pass an old state once we can make use of it.
[...summaryInputs, ...linkedInputs],
parsedArgs['enable-experiment'] as List<String>,
void onDiagnostic(fe.DiagnosticMessage message) {
fe.printDiagnosticMessage(message, out.writeln);
succeeded = false;
List<int> kernel;
bool wroteUsedDills = false;
if (usingIncrementalCompiler) {
state.options.onDiagnostic = onDiagnostic;
Component incrementalComponent = await state.incrementalCompiler
.computeDelta(entryPoints: sources, fullComponent: true);
if (recordUsedInputs) {
Set<Uri> usedOutlines = {};
for (Library lib in state.incrementalCompiler.neededDillLibraries) {
if (lib.importUri.scheme == "dart") continue;
Uri uri = state.libraryToInputDill[lib.importUri];
if (uri == null) {
throw new StateError("Library ${lib.importUri} was recorded as used, "
"but was not in the list of known libraries.");
var outputUsedFile = new File(parsedArgs["used-inputs"]);
outputUsedFile.createSync(recursive: true);
wroteUsedDills = true;
kernel = await state.incrementalCompiler.context.runInContext((_) {
if (summaryOnly) {
incrementalComponent.problemsAsJson = null;
null, true, incrementalComponent.mode);
return Future.value(fe.serializeComponent(incrementalComponent,
includeSources: false, includeOffsets: false));
return Future.value(fe.serializeComponent(incrementalComponent,
filter: excludeNonSources
? (library) => sources.contains(library.importUri)
: null,
includeOffsets: true));
} else if (summaryOnly) {
kernel = await fe.compileSummary(state, sources, onDiagnostic,
includeOffsets: false);
} else {
Component component =
await fe.compileComponent(state, sources, onDiagnostic);
kernel = fe.serializeComponent(component,
filter: excludeNonSources
? (library) => sources.contains(library.importUri)
: null,
includeOffsets: true);
state.options.onDiagnostic = null; // See
if (!wroteUsedDills && recordUsedInputs) {
// The path taken didn't record inputs used: Say we used everything.
var outputUsedFile = new File(parsedArgs["used-inputs"]);
outputUsedFile.createSync(recursive: true);
Set<Uri> allFiles = {...summaryInputs, ...linkedInputs};
wroteUsedDills = true;
if (kernel != null) {
var outputFile = new File(parsedArgs['output']);
outputFile.createSync(recursive: true);
} else {
return new ComputeKernelResult(succeeded, state);
/// Make sure the output is stable by sorting libraries and additional exports.
void makeStable(Component c) {
// Make sure the output is stable.
c.libraries.sort((l1, l2) {
return "${l1.fileUri}".compareTo("${l2.fileUri}");
for (Library library in c.libraries) {
library.additionalExports.sort((Reference r1, Reference r2) {
return "${r1.canonicalName}".compareTo("${r2.canonicalName}");
/// Extends the DevCompilerTarget to transform outlines to meet the requirements
/// of summaries in bazel and package-build.
/// Build systems like package-build may provide the same input file twice to
/// the summary worker, but only intends to have it in one output summary. The
/// convention is that if it is listed as a source, it is intended to be part of
/// the output, if the source file was loaded as a dependency, then it was
/// already included in a different summary. The transformation below ensures
/// that the output summary doesn't include those implicit inputs.
/// Note: this transformation is destructive and is only intended to be used
/// when generating summaries.
class DevCompilerSummaryTarget extends DevCompilerTarget {
final List<Uri> sources;
final bool excludeNonSources;
this.sources, this.excludeNonSources, TargetFlags targetFlags)
: super(targetFlags);
void performOutlineTransformations(Component component) {
if (!excludeNonSources) return;
List<Library> libraries = new List.from(component.libraries);
Set<Uri> include = sources.toSet();
for (var lib in libraries) {
if (include.contains(lib.importUri)) {
} else {
// Excluding the library also means that their canonical names will not
// be computed as part of serialization, so we need to do that
// preemtively here to avoid errors when serializing references to
// elements of these libraries.
Uri _toUri(String uriString) {
if (uriString == null) return null;
// Windows-style paths use '\', so convert them to '/' in case they've been
// concatenated with Unix-style paths.
return Uri.base.resolve(uriString.replaceAll("\\", "/"));
Map<String, String> _parseEnvironmentDefines(List<String> args) {
var environment = <String, String>{};
for (var arg in args) {
var eq = arg.indexOf('=');
if (eq <= 0) {
var kind = eq == 0 ? 'name' : 'value';
throw FormatException('no $kind given to -D option `$arg`');
var name = arg.substring(0, eq);
var value = arg.substring(eq + 1);
environment[name] = value;
return environment;