Allow to specify what dill file to initialize from
This CL updates the frontend_server to allow specifying what dill
file to initialize from.
Normally, it tries to initialize from the same dill file as is the
output. This is fine in many cases (e.g. flutter run always says it
should output to build/app.dill): The first time it compiles
everything, subsequent times (across restarts) it only recompiles
what's needed.
When running tests, however, the output is in a temporary directory,
so it doesn't work across restarts. The startup time for tests is
always rather high, because it always has to recompile everything.
This CL updates the frontend_server to allow specifying what dill
file to initialize from. Flutter can thus save the compiled test
file in a centralized location (e.g. build/test.dill) and load from
that across restarts.
Plumbing this into flutter yields the following result, when running
a single test (the one automatically created when creating a new
flutter project):
Before: ~5.5 seconds
Now, first run: ~5.5 second (unchanged)
Now, subsequent runs: ~3.4 seconds
Approximate breakdown:
Startup cost (e.g. if there's no test directory): ~ 800 ms
Starting up the actual frontend server: ~150 ms
Reading source from old dill, loading all relevant files
and invalidating source: ~ 300 ms
Compiling everything (or, rather, nothing as it's all
from the dill at this point): ~ 650 ms
Serializing: ~ 250 ms
Copying files: ~ 50 ms
Running the actual test: ~1100 ms
Total: ~3300 ms
`time flutter test --local-engine=host_debug --preview-dart-2`
says ~3.4 seconds...
To compare, --no-preview-dart-2 takes ~2.2 seconds,
so dart2 still has a (significantly) higher startup cost.
Moves the needle on https://github.com/flutter/flutter/issues/15767.
Change-Id: I36a2d05bc76e0193d369df42eb3b9f08416dd78f
Reviewed-on: https://dart-review.googlesource.com/51820
Reviewed-by: Kevin Millikin <kmillikin@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
Commit-Queue: Jens Johansen <jensj@google.com>
diff --git a/pkg/front_end/lib/src/api_prototype/incremental_kernel_generator.dart b/pkg/front_end/lib/src/api_prototype/incremental_kernel_generator.dart
index 768cb71..f10c806 100644
--- a/pkg/front_end/lib/src/api_prototype/incremental_kernel_generator.dart
+++ b/pkg/front_end/lib/src/api_prototype/incremental_kernel_generator.dart
@@ -19,10 +19,10 @@
abstract class IncrementalKernelGenerator {
factory IncrementalKernelGenerator(CompilerOptions options, Uri entryPoint,
- [Uri bootstrapDill]) {
+ [Uri initializeFromDillUri]) {
return new IncrementalCompiler(
new CompilerContext(new ProcessedOptions(options, false, [entryPoint])),
- bootstrapDill);
+ initializeFromDillUri);
}
/// Returns a component whose libraries are the recompiled libraries,
diff --git a/pkg/kernel/lib/binary/ast_from_binary.dart b/pkg/kernel/lib/binary/ast_from_binary.dart
index 8e5c85a..0b8053d 100644
--- a/pkg/kernel/lib/binary/ast_from_binary.dart
+++ b/pkg/kernel/lib/binary/ast_from_binary.dart
@@ -402,6 +402,22 @@
}
}
+ /// Deserializes the source and stores it in [component].
+ ///
+ /// The input bytes may contain multiple files concatenated.
+ void readComponentSource(Component component) {
+ List<int> componentFileSizes = _indexComponents();
+ if (componentFileSizes.length > 1) {
+ _disableLazyReading = true;
+ }
+ int componentFileIndex = 0;
+ while (_byteOffset < _bytes.length) {
+ _readOneComponentSource(
+ component, componentFileSizes[componentFileIndex]);
+ ++componentFileIndex;
+ }
+ }
+
/// Reads a single component file from the input and loads it into [component],
/// overwriting and reusing any existing data in the component.
///
@@ -468,6 +484,31 @@
return result;
}
+ void _readOneComponentSource(Component component, int componentFileSize) {
+ _componentStartOffset = _byteOffset;
+
+ final int magic = readUint32();
+ if (magic != Tag.ComponentFile) {
+ throw fail('This is not a binary dart file. '
+ 'Magic number was: ${magic.toRadixString(16)}');
+ }
+
+ final int formatVersion = readUint32();
+ if (formatVersion != Tag.BinaryFormatVersion) {
+ throw fail('Invalid kernel binary format version '
+ '(found ${formatVersion}, expected ${Tag.BinaryFormatVersion})');
+ }
+
+ // Read component index from the end of this ComponentFiles serialized data.
+ _ComponentIndex index = _readComponentIndex(componentFileSize);
+
+ _byteOffset = index.binaryOffsetForSourceTable;
+ Map<Uri, Source> uriToSource = readUriToSource();
+ component.uriToSource.addAll(uriToSource);
+
+ _byteOffset = _componentStartOffset + componentFileSize;
+ }
+
void _readOneComponent(Component component, int componentFileSize) {
_componentStartOffset = _byteOffset;
diff --git a/pkg/kernel/lib/kernel.dart b/pkg/kernel/lib/kernel.dart
index 5ffbd41..aa2a903 100644
--- a/pkg/kernel/lib/kernel.dart
+++ b/pkg/kernel/lib/kernel.dart
@@ -33,6 +33,12 @@
return component;
}
+Component loadComponentSourceFromBytes(List<int> bytes, [Component component]) {
+ component ??= new Component();
+ new BinaryBuilder(bytes).readComponentSource(component);
+ return component;
+}
+
Future writeComponentToBinary(Component component, String path) {
var sink;
if (path == 'null' || path == 'stdout') {
diff --git a/pkg/vm/lib/frontend_server.dart b/pkg/vm/lib/frontend_server.dart
index 450a03e..c38213d 100644
--- a/pkg/vm/lib/frontend_server.dart
+++ b/pkg/vm/lib/frontend_server.dart
@@ -25,7 +25,8 @@
import 'package:kernel/ast.dart';
import 'package:kernel/binary/ast_to_binary.dart';
import 'package:kernel/binary/limited_ast_to_binary.dart';
-import 'package:kernel/kernel.dart' show Component, loadComponentFromBytes;
+import 'package:kernel/kernel.dart'
+ show Component, loadComponentSourceFromBytes;
import 'package:kernel/target/targets.dart';
import 'package:path/path.dart' as path;
import 'package:usage/uuid/uuid.dart';
@@ -88,7 +89,12 @@
' option',
defaultsTo: 'org-dartlang-root',
hide: true)
- ..addFlag('verbose', help: 'Enables verbose output from the compiler.');
+ ..addFlag('verbose', help: 'Enables verbose output from the compiler.')
+ ..addOption('initialize-from-dill',
+ help: 'Normally the output dill is used to specify which dill to '
+ 'initialize from, but it can be overwritten here.',
+ defaultsTo: null,
+ hide: true);
String usage = '''
Usage: server [options] [input.dart]
@@ -179,6 +185,7 @@
String _kernelBinaryFilename;
String _kernelBinaryFilenameIncremental;
String _kernelBinaryFilenameFull;
+ String _initializeFromDill;
final ProgramTransformer transformer;
@@ -203,6 +210,8 @@
? '${_options["output-dill"]}.incremental.dill'
: '$filename.incremental.dill');
_kernelBinaryFilename = _kernelBinaryFilenameFull;
+ _initializeFromDill =
+ _options['initialize-from-dill'] ?? _kernelBinaryFilenameFull;
final String boundaryKey = new Uuid().generateV4();
_outputStream.writeln('result $boundaryKey');
final Uri sdkRoot = _ensureFolderPath(options['sdk-root']);
@@ -270,9 +279,9 @@
Component component;
if (options['incremental']) {
_compilerOptions = compilerOptions;
- _generator = generator ??
- _createGenerator(new Uri.file(_kernelBinaryFilenameFull));
- await invalidateIfBootstrapping();
+ _generator =
+ generator ?? _createGenerator(new Uri.file(_initializeFromDill));
+ await invalidateIfInitializingFromDill();
component = await _runWithPrintRedirection(() => _generator.compile());
} else {
if (options['link-platform']) {
@@ -321,18 +330,19 @@
await sink.close();
}
- Future<Null> invalidateIfBootstrapping() async {
+ Future<Null> invalidateIfInitializingFromDill() async {
if (_kernelBinaryFilename != _kernelBinaryFilenameFull) return null;
- // If the generator is initialized bootstrapping is not in effect anyway,
- // so there's no reason to spend time invalidating what should be
- // invalidated by the normal approach anyway.
+ // If the generator is initialized, it's not going to initialize from dill
+ // again anyway, so there's no reason to spend time invalidating what should
+ // be invalidated by the normal approach anyway.
if (_generator.initialized) return null;
try {
- final File f = new File(_kernelBinaryFilenameFull);
+ final File f = new File(_initializeFromDill);
if (!f.existsSync()) return null;
- final Component component = loadComponentFromBytes(f.readAsBytesSync());
+ final Component component =
+ loadComponentSourceFromBytes(f.readAsBytesSync());
for (Uri uri in component.uriToSource.keys) {
if ('$uri' == '') continue;
@@ -357,8 +367,8 @@
}
} catch (e) {
// If there's a failure in the above block we might not have invalidated
- // correctly. Create a new generator that doesn't bootstrap to avoid missing
- // any changes.
+ // correctly. Create a new generator that doesn't initialize from dill to
+ // avoid missing any changes.
_generator = _createGenerator(null);
}
}
@@ -367,7 +377,7 @@
Future<Null> recompileDelta({String filename}) async {
final String boundaryKey = new Uuid().generateV4();
_outputStream.writeln('result $boundaryKey');
- await invalidateIfBootstrapping();
+ await invalidateIfInitializingFromDill();
if (filename != null) {
setMainSourceFilename(filename);
}
@@ -421,9 +431,9 @@
return Uri.base.resolveUri(new Uri.file(fileOrUri));
}
- IncrementalCompiler _createGenerator(Uri bootstrapDill) {
+ IncrementalCompiler _createGenerator(Uri initializeFromDillUri) {
return new IncrementalCompiler(_compilerOptions, _mainSource,
- bootstrapDill: bootstrapDill);
+ initializeFromDillUri: initializeFromDillUri);
}
Uri _ensureFolderPath(String path) {
diff --git a/pkg/vm/lib/incremental_compiler.dart b/pkg/vm/lib/incremental_compiler.dart
index 3541bc2..f52a72a 100644
--- a/pkg/vm/lib/incremental_compiler.dart
+++ b/pkg/vm/lib/incremental_compiler.dart
@@ -21,11 +21,12 @@
CompilerOptions _compilerOptions;
bool initialized = false;
bool fullComponent = false;
+ Uri initializeFromDillUri;
IncrementalCompiler(this._compilerOptions, Uri entryPoint,
- {Uri bootstrapDill}) {
+ {this.initializeFromDillUri}) {
_generator = new IncrementalKernelGenerator(
- _compilerOptions, entryPoint, bootstrapDill);
+ _compilerOptions, entryPoint, initializeFromDillUri);
_pendingDeltas = <Component>[];
}
