| // Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| /// A library to invoke the CFE to compute kernel summary files. |
| /// |
| /// Used by `utils/bazel/kernel_worker.dart`. |
| import 'dart:async'; |
| import 'dart:io'; |
| |
| import 'package:_fe_analyzer_shared/src/macros/executor/isolated_executor.dart' |
| as isolatedExecutor; |
| import 'package:_fe_analyzer_shared/src/macros/executor/process_executor.dart' |
| as processExecutor; |
| import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart' |
| show SerializationMode; |
| import 'package:args/args.dart'; |
| import 'package:build_integration/file_system/multi_root.dart'; |
| import 'package:compiler/src/kernel/dart2js_target.dart'; |
| import 'package:dev_compiler/src/kernel/target.dart'; |
| import 'package:front_end/src/api_prototype/incremental_kernel_generator.dart'; |
| import 'package:front_end/src/api_unstable/bazel_worker.dart' as fe; |
| import 'package:front_end/src/fasta/kernel/macro/macro.dart'; |
| import 'package:kernel/ast.dart' show Component, Library, Reference; |
| import 'package:kernel/target/targets.dart'; |
| import 'package:vm/target/flutter.dart'; |
| import 'package:vm/target/flutter_runner.dart'; |
| import 'package:vm/target/vm.dart'; |
| |
| /// If the last arg starts with `@`, this reads the file it points to and treats |
| /// each line as an additional arg. |
| /// |
| /// This is how individual work request args are differentiated from startup |
| /// args in bazel (individual work request args go in that file). |
| List<String> preprocessArgs(List<String> args) { |
| args = new List.from(args); |
| if (args.isEmpty) { |
| return args; |
| } |
| String lastArg = args.last; |
| if (lastArg.startsWith('@')) { |
| File argsFile = new File(lastArg.substring(1)); |
| try { |
| args.removeLast(); |
| args.addAll(argsFile.readAsLinesSync()); |
| } on FileSystemException catch (e) { |
| throw new Exception('Failed to read file specified by $lastArg : $e'); |
| } |
| } |
| return args; |
| } |
| |
| /// An [ArgParser] for generating kernel summaries. |
| final summaryArgsParser = new ArgParser() |
| ..addFlag('help', negatable: false, abbr: 'h') |
| ..addFlag('exclude-non-sources', |
| negatable: false, |
| help: 'Whether source files loaded implicitly should be included as ' |
| 'part of the summary.') |
| ..addFlag('summary-only', |
| defaultsTo: true, |
| negatable: true, |
| help: 'Whether to only build summary files.') |
| ..addFlag('summary', |
| defaultsTo: true, |
| negatable: true, |
| help: 'Whether or not to build summary files.') |
| ..addOption('target', |
| allowed: const [ |
| 'vm', |
| 'flutter', |
| 'flutter_runner', |
| 'dart2js', |
| 'dart2js_summary', |
| 'ddc', |
| ], |
| help: 'Build kernel for the vm, flutter, flutter_runner, dart2js or ddc') |
| ..addOption('dart-sdk-summary') |
| ..addMultiOption('input-summary') |
| ..addMultiOption('input-linked') |
| ..addMultiOption('multi-root') |
| ..addOption('multi-root-scheme', defaultsTo: 'org-dartlang-multi-root') |
| ..addOption('libraries-file') |
| ..addOption('packages-file') |
| ..addMultiOption('source') |
| ..addOption('output') |
| ..addFlag('reuse-compiler-result', defaultsTo: false) |
| ..addFlag('use-incremental-compiler', defaultsTo: false) |
| ..addOption('used-inputs') |
| ..addFlag('track-widget-creation', defaultsTo: false) |
| ..addMultiOption('enable-experiment', |
| help: 'Enable a language experiment when invoking the CFE.') |
| ..addMultiOption('define', abbr: 'D') |
| ..addFlag('verbose', defaultsTo: false) |
| ..addFlag('sound-null-safety', defaultsTo: false) |
| ..addOption('verbosity', |
| defaultsTo: fe.Verbosity.defaultValue, |
| help: 'Sets the verbosity level used for filtering messages during ' |
| 'compilation.', |
| allowed: fe.Verbosity.allowedValues, |
| allowedHelp: fe.Verbosity.allowedValuesHelp) |
| ..addMultiOption('precompiled-macro', |
| help: 'Configuration for precompiled macro binaries or kernel files.\n' |
| 'Must be used in combination with --precompiled-macro-format.\n' |
| 'The expected format of this option is as follows: ' |
| '<macro-library-uri>;<absolute-path-to-binary>\nFor example: ' |
| '--precompiled-macro="package:some_macro/some_macro.dart;' |
| '/path/to/compiled/macro"') |
| ..addOption('precompiled-macro-format', |
| help: 'The format for precompiled macros.', |
| allowed: ['aot', 'kernel'], |
| defaultsTo: 'aot') |
| ..addOption('macro-serialization-mode', |
| help: 'The serialization mode for communicating with macros.', |
| allowed: ['bytedata', 'json'], |
| defaultsTo: 'bytedata'); |
| |
| class ComputeKernelResult { |
| final bool succeeded; |
| final fe.InitializedCompilerState? previousState; |
| |
| ComputeKernelResult(this.succeeded, this.previousState); |
| } |
| |
| /// Computes a kernel file based on [args]. |
| /// |
| /// If [isWorker] is true then exit codes will not be set on failure. |
| /// |
| /// If [outputBuffer] is provided then messages will be written to that buffer |
| /// instead of printed to the console. |
| /// |
| /// Returns whether or not the summary was successfully output. |
| Future<ComputeKernelResult> computeKernel(List<String> args, |
| {bool isWorker: false, |
| StringBuffer? outputBuffer, |
| Map<Uri, List<int>>? inputDigests, |
| fe.InitializedCompilerState? previousState}) async { |
| inputDigests ??= <Uri, List<int>>{}; |
| dynamic out = outputBuffer ?? stderr; |
| bool succeeded = true; |
| |
| var parsedArgs = summaryArgsParser.parse(args); |
| |
| if (parsedArgs['help']) { |
| out.writeln(summaryArgsParser.usage); |
| if (!isWorker) exit(0); |
| return new ComputeKernelResult(false, previousState); |
| } |
| |
| // Bazel creates an overlay file system where some files may be located in the |
| // source tree, some in a gendir, and some in a bindir. The multi-root file |
| // system hides this from the front end. |
| var multiRoots = parsedArgs['multi-root'].map(Uri.base.resolve).toList(); |
| if (multiRoots.isEmpty) multiRoots.add(Uri.base); |
| var fileSystem = new MultiRootFileSystem(parsedArgs['multi-root-scheme'], |
| multiRoots, fe.StandardFileSystem.instance); |
| var sources = (parsedArgs['source'] as List<String>).map(toUri).toList(); |
| var excludeNonSources = parsedArgs['exclude-non-sources'] as bool; |
| |
| var nnbdMode = parsedArgs['sound-null-safety'] as bool |
| ? fe.NnbdMode.Strong |
| : fe.NnbdMode.Weak; |
| var summaryOnly = parsedArgs['summary-only'] as bool; |
| var summary = parsedArgs['summary'] as bool; |
| if (summaryOnly && !summary) { |
| throw new ArgumentError('--summary-only conflicts with --no-summary'); |
| } |
| var trackWidgetCreation = parsedArgs['track-widget-creation'] as bool; |
| |
| // TODO(sigmund,jakemac): make target mandatory. We allow null to be backwards |
| // compatible while we migrate existing clients of this tool. |
| var targetName = |
| (parsedArgs['target'] as String?) ?? (summaryOnly ? 'ddc' : 'vm'); |
| var targetFlags = new TargetFlags( |
| trackWidgetCreation: trackWidgetCreation, |
| enableNullSafety: nnbdMode == fe.NnbdMode.Strong); |
| Target target; |
| switch (targetName) { |
| case 'vm': |
| target = new VmTarget(targetFlags); |
| if (summaryOnly) { |
| out.writeln('error: --summary-only not supported for the vm target'); |
| } |
| break; |
| case 'flutter': |
| target = new FlutterTarget(targetFlags); |
| if (summaryOnly) { |
| throw new ArgumentError( |
| 'error: --summary-only not supported for the flutter target'); |
| } |
| break; |
| case 'flutter_runner': |
| target = new FlutterRunnerTarget(targetFlags); |
| if (summaryOnly) { |
| throw new ArgumentError('error: --summary-only not supported for the ' |
| 'flutter_runner target'); |
| } |
| break; |
| case 'dart2js': |
| target = new Dart2jsTarget('dart2js', targetFlags); |
| if (summaryOnly) { |
| out.writeln( |
| 'error: --summary-only not supported for the dart2js target'); |
| } |
| break; |
| case 'dart2js_summary': |
| target = new Dart2jsSummaryTarget( |
| 'dart2js', sources, excludeNonSources, targetFlags); |
| if (!summaryOnly) { |
| out.writeln( |
| 'error: --no-summary-only not supported for the dart2js summary target'); |
| } |
| break; |
| case 'ddc': |
| // TODO(jakemac):If `generateKernel` changes to return a summary |
| // component, process the component instead. |
| target = |
| new DevCompilerSummaryTarget(sources, excludeNonSources, targetFlags); |
| if (!summaryOnly) { |
| out.writeln('error: --no-summary-only not supported for the ' |
| 'ddc target'); |
| } |
| break; |
| default: |
| out.writeln('error: unsupported target: $targetName'); |
| return ComputeKernelResult(false, previousState); |
| } |
| |
| List<Uri> linkedInputs = |
| (parsedArgs['input-linked'] as List<String>).map(toUri).toList(); |
| |
| List<Uri> summaryInputs = |
| (parsedArgs['input-summary'] as List<String>).map(toUri).toList(); |
| |
| fe.InitializedCompilerState state; |
| bool usingIncrementalCompiler = false; |
| bool recordUsedInputs = parsedArgs["used-inputs"] != null; |
| var environmentDefines = _parseEnvironmentDefines(parsedArgs['define']); |
| var verbose = parsedArgs['verbose'] as bool; |
| var verbosity = fe.Verbosity.parseArgument(parsedArgs['verbosity']); |
| Uri? sdkSummaryUri = toUriNullable(parsedArgs['dart-sdk-summary']); |
| |
| if (parsedArgs['use-incremental-compiler']) { |
| usingIncrementalCompiler = true; |
| |
| // If digests weren't given and if not in worker mode, create fake data and |
| // ensure we don't have a previous state (as that wouldn't be safe with |
| // fake input digests). |
| if (!isWorker && inputDigests.isEmpty) { |
| previousState = null; |
| if (sdkSummaryUri != null) { |
| inputDigests[sdkSummaryUri] = const [0]; |
| } |
| for (Uri uri in summaryInputs) { |
| inputDigests[uri] = const [0]; |
| } |
| for (Uri uri in linkedInputs) { |
| inputDigests[uri] = const [0]; |
| } |
| } |
| |
| state = await fe.initializeIncrementalCompiler( |
| previousState, |
| { |
| "target=$targetName", |
| "trackWidgetCreation=$trackWidgetCreation", |
| "multiRootScheme=${fileSystem.markerScheme}", |
| "multiRootRoots=${fileSystem.roots}", |
| }, |
| sdkSummaryUri, |
| toUriNullable(parsedArgs['packages-file']), |
| toUriNullable(parsedArgs['libraries-file']), |
| [...summaryInputs, ...linkedInputs], |
| inputDigests, |
| target, |
| fileSystem, |
| (parsedArgs['enable-experiment'] as List<String>), |
| summaryOnly, |
| environmentDefines, |
| trackNeededDillLibraries: recordUsedInputs, |
| verbose: verbose, |
| nnbdMode: nnbdMode); |
| } else { |
| state = fe.initializeCompiler( |
| // TODO(sigmund): pass an old state once we can make use of it. |
| null, |
| sdkSummaryUri, |
| toUriNullable(parsedArgs['libraries-file']), |
| toUriNullable(parsedArgs['packages-file']), |
| [...summaryInputs, ...linkedInputs], |
| target, |
| fileSystem, |
| parsedArgs['enable-experiment'] as List<String>, |
| environmentDefines, |
| verbose: verbose, |
| nnbdMode: nnbdMode); |
| } |
| |
| // Either set up or reset the state for macros based on experiment status. |
| // TODO: Make this a part of `initializeCompiler`, if/when we want to make it |
| // more widely supported. |
| if (state.processedOpts.globalFeatures.macros.isEnabled) { |
| enableMacros = true; |
| forceEnableMacros = true; |
| |
| SerializationMode serializationMode; |
| switch (parsedArgs['macro-serialization-mode']) { |
| case 'json': |
| serializationMode = SerializationMode.jsonServer; |
| break; |
| case 'bytedata': |
| serializationMode = SerializationMode.byteDataServer; |
| break; |
| default: |
| throw ArgumentError('Unrecognized macro serialization mode ' |
| '${parsedArgs['macro-serialization-mode']}'); |
| } |
| |
| // TODO: Handle invalidation of precompiled macros. |
| // TODO: Handle multiple macro libraries compiled to a single precompiled |
| // kernel file. |
| var macroExecutor = state.processedOpts.macroExecutor; |
| var format = parsedArgs['precompiled-macro-format']; |
| for (var parts in (parsedArgs['precompiled-macro'] as List<String>) |
| .map((arg) => arg.split(';'))) { |
| var library = Uri.parse(parts[0]); |
| if (macroExecutor.libraryIsRegistered(library)) { |
| continue; |
| } |
| var programUri = toUri(parts[1]); |
| switch (format) { |
| case 'kernel': |
| macroExecutor.registerExecutorFactory( |
| () => isolatedExecutor.start(serializationMode, programUri), |
| {library}); |
| break; |
| case 'aot': |
| macroExecutor.registerExecutorFactory( |
| () => processExecutor.start( |
| serializationMode, |
| processExecutor.CommunicationChannel.socket, |
| programUri.toFilePath()), |
| {library}); |
| break; |
| default: |
| throw ArgumentError('Unrecognized precompiled macro format $format'); |
| } |
| } |
| } else { |
| enableMacros = false; |
| forceEnableMacros = false; |
| await state.options.macroExecutor?.close(); |
| state.options.macroExecutor = null; |
| } |
| |
| void onDiagnostic(fe.DiagnosticMessage message) { |
| if (fe.Verbosity.shouldPrint(verbosity, message)) { |
| fe.printDiagnosticMessage(message, out.writeln); |
| } |
| if (message.severity == fe.Severity.error) { |
| succeeded = false; |
| } |
| } |
| |
| List<int>? kernel; |
| bool wroteUsedDills = false; |
| if (usingIncrementalCompiler) { |
| state.options.onDiagnostic = onDiagnostic; |
| IncrementalCompilerResult incrementalCompilerResult = |
| await state.incrementalCompiler!.computeDelta( |
| entryPoints: sources, |
| fullComponent: true, |
| trackNeededDillLibraries: recordUsedInputs); |
| Component incrementalComponent = incrementalCompilerResult.component; |
| |
| if (recordUsedInputs) { |
| Set<Uri> usedOutlines = {}; |
| for (Library lib in incrementalCompilerResult.neededDillLibraries!) { |
| if (lib.importUri.isScheme("dart")) continue; |
| Uri? uri = state.libraryToInputDill![lib.importUri]; |
| if (uri == null) { |
| throw new StateError("Library ${lib.importUri} was recorded as used, " |
| "but was not in the list of known libraries."); |
| } |
| usedOutlines.add(uri); |
| } |
| var outputUsedFile = new File(parsedArgs["used-inputs"]); |
| outputUsedFile.createSync(recursive: true); |
| outputUsedFile.writeAsStringSync(usedOutlines.join("\n")); |
| wroteUsedDills = true; |
| } |
| |
| kernel = await state.incrementalCompiler!.context.runInContext((_) { |
| if (summaryOnly) { |
| incrementalComponent.uriToSource.clear(); |
| incrementalComponent.problemsAsJson = null; |
| incrementalComponent.setMainMethodAndMode( |
| null, true, incrementalComponent.mode); |
| target.performOutlineTransformations(incrementalComponent); |
| makeStable(incrementalComponent); |
| return Future.value(fe.serializeComponent(incrementalComponent, |
| includeSources: false, includeOffsets: false)); |
| } |
| |
| makeStable(incrementalComponent); |
| |
| return Future.value(fe.serializeComponent(incrementalComponent, |
| filter: excludeNonSources |
| ? (library) => sources.contains(library.importUri) |
| : null, |
| includeOffsets: true)); |
| }); |
| } else if (summaryOnly) { |
| kernel = await fe.compileSummary(state, sources, onDiagnostic, |
| includeOffsets: false); |
| } else { |
| Component? component = await fe |
| .compileComponent(state, sources, onDiagnostic, buildSummary: summary); |
| if (component != null) { |
| kernel = fe.serializeComponent(component, |
| filter: excludeNonSources |
| ? (library) => sources.contains(library.importUri) |
| : null, |
| includeOffsets: true); |
| } |
| } |
| state.options.onDiagnostic = null; // See http://dartbug.com/36983. |
| |
| if (!wroteUsedDills && recordUsedInputs) { |
| // The path taken didn't record inputs used: Say we used everything. |
| var outputUsedFile = new File(parsedArgs["used-inputs"]); |
| outputUsedFile.createSync(recursive: true); |
| Set<Uri> allFiles = {...summaryInputs, ...linkedInputs}; |
| outputUsedFile.writeAsStringSync(allFiles.join("\n")); |
| wroteUsedDills = true; |
| } |
| |
| if (kernel != null) { |
| var outputFile = new File(parsedArgs['output']); |
| outputFile.createSync(recursive: true); |
| outputFile.writeAsBytesSync(kernel); |
| } else { |
| assert(!succeeded); |
| } |
| |
| return new ComputeKernelResult(succeeded, state); |
| } |
| |
| /// Make sure the output is stable by sorting libraries and additional exports. |
| void makeStable(Component c) { |
| // Make sure the output is stable. |
| c.libraries.sort((l1, l2) { |
| return "${l1.fileUri}".compareTo("${l2.fileUri}"); |
| }); |
| c.problemsAsJson?.sort(); |
| c.computeCanonicalNames(); |
| for (Library library in c.libraries) { |
| library.additionalExports.sort((Reference r1, Reference r2) { |
| return "${r1.canonicalName}".compareTo("${r2.canonicalName}"); |
| }); |
| library.problemsAsJson?.sort(); |
| } |
| } |
| |
| class DevCompilerSummaryTarget extends DevCompilerTarget with SummaryMixin { |
| final List<Uri> sources; |
| final bool excludeNonSources; |
| |
| DevCompilerSummaryTarget( |
| this.sources, this.excludeNonSources, TargetFlags targetFlags) |
| : super(targetFlags); |
| } |
| |
| Uri? toUriNullable(String? uriString) { |
| if (uriString == null) return null; |
| return toUri(uriString); |
| } |
| |
| Uri toUri(String uriString) { |
| // Windows-style paths use '\', so convert them to '/' in case they've been |
| // concatenated with Unix-style paths. |
| return Uri.base.resolve(uriString.replaceAll("\\", "/")); |
| } |
| |
| Map<String, String> _parseEnvironmentDefines(List<String> args) { |
| var environment = <String, String>{}; |
| |
| for (var arg in args) { |
| var eq = arg.indexOf('='); |
| if (eq <= 0) { |
| var kind = eq == 0 ? 'name' : 'value'; |
| throw FormatException('no $kind given to -D option `$arg`'); |
| } |
| var name = arg.substring(0, eq); |
| var value = arg.substring(eq + 1); |
| environment[name] = value; |
| } |
| |
| return environment; |
| } |
