Introduce option to serialize less in frontend_server
This CL introduces an option - unsafe-package-serialization - where a
caller can select to reuse the serialization of packages. This makes it
unsafe in general, but if never invalidating packages, such as in tests,
it can be used to greatly increase the speed of processing many inputs.
This change - used in flutter - takes the time it takes to run
`flutter test` in `packages/flutter` from ~2 minutes 30 seconds to
~1 minute 15 seconds (on my machine).
A change to how flutter test executes the frontend_server
(to pass the option) is needed for it to take effect though.
Change-Id: Ibe47b43c41286eb08f80c9e7a398e1dc67bf9bc0
Reviewed-on: https://dart-review.googlesource.com/48200
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
diff --git a/pkg/vm/lib/frontend_server.dart b/pkg/vm/lib/frontend_server.dart
index 418ce4b..00cfb12 100644
--- a/pkg/vm/lib/frontend_server.dart
+++ b/pkg/vm/lib/frontend_server.dart
@@ -101,7 +101,15 @@
help: 'Includes sources into generated dill file. Having sources'
' allows to effectively use observatory to debug produced'
' application, produces better stack traces on exceptions.',
- defaultsTo: true);
+ defaultsTo: true)
+ ..addFlag('unsafe-package-serialization',
+ help: 'Potentially unsafe: Does not allow for invalidating packages, '
+ 'additionally the output dill file might include more libraries than '
+ 'needed. The use case is test-runs, where invalidation is not really '
+ 'used, and where dill filesize does not matter, and the gain is '
+ 'improved speed.',
+ defaultsTo: false,
+ hide: true);
String usage = '''
Usage: server [options] [input.dart]
@@ -201,7 +209,7 @@
/// Class that for test mocking purposes encapsulates creation of [BinaryPrinter].
class BinaryPrinterFactory {
/// Creates new [BinaryPrinter] to write to [targetSink].
- BinaryPrinter newBinaryPrinter(IOSink targetSink) {
+ BinaryPrinter newBinaryPrinter(Sink<List<int>> targetSink) {
return new LimitedBinaryPrinter(targetSink, (_) => true /* predicate */,
false /* excludeUriToSource */);
}
@@ -209,13 +217,16 @@
class FrontendCompiler implements CompilerInterface {
FrontendCompiler(this._outputStream,
- {this.printerFactory, this.transformer}) {
+ {this.printerFactory,
+ this.transformer,
+ this.unsafePackageSerialization}) {
_outputStream ??= stdout;
printerFactory ??= new BinaryPrinterFactory();
}
StringSink _outputStream;
BinaryPrinterFactory printerFactory;
+ bool unsafePackageSerialization;
CompilerOptions _compilerOptions;
Uri _mainSource;
@@ -388,6 +399,9 @@
return "${r1.canonicalName}".compareTo("${r2.canonicalName}");
});
}
+ if (unsafePackageSerialization == true) {
+ writePackagesToSinkAndTrimComponent(component, sink);
+ }
printer.writeComponentFile(component);
await sink.close();
@@ -495,6 +509,63 @@
_outputStream.writeln(boundaryKey);
}
+ /// Map of already serialized dill data. All uris in a serialized component
+ /// maps to the same blob of data. Used by
+ /// [writePackagesToSinkAndTrimComponent].
+ Map<Uri, List<int>> cachedPackageLibraries = new Map<Uri, List<int>>();
+
+ writePackagesToSinkAndTrimComponent(
+ Component deltaProgram, Sink<List<int>> ioSink) {
+ if (deltaProgram == null) return;
+
+ List<Library> packageLibraries = new List<Library>();
+ List<Library> libraries = new List<Library>();
+ deltaProgram.computeCanonicalNames();
+
+ for (var lib in deltaProgram.libraries) {
+ Uri uri = lib.importUri;
+ if (uri.scheme == "package") {
+ packageLibraries.add(lib);
+ } else {
+ libraries.add(lib);
+ }
+ }
+ deltaProgram.libraries
+ ..clear()
+ ..addAll(libraries);
+
+ Map<String, List<Library>> newPackages = new Map<String, List<Library>>();
+ Set<List<int>> alreadyAdded = new Set<List<int>>();
+ for (Library lib in packageLibraries) {
+ List<int> data = cachedPackageLibraries[lib.fileUri];
+ if (data != null) {
+ if (alreadyAdded.add(data)) {
+ ioSink.add(data);
+ }
+ } else {
+ String package = lib.importUri.pathSegments.first;
+ newPackages[package] ??= <Library>[];
+ newPackages[package].add(lib);
+ }
+ }
+
+ for (String package in newPackages.keys) {
+ List<Library> libraries = newPackages[package];
+ Component singleLibrary = new Component(
+ libraries: libraries,
+ uriToSource: deltaProgram.uriToSource,
+ nameRoot: deltaProgram.root);
+ ByteSink byteSink = new ByteSink();
+ final BinaryPrinter printer = printerFactory.newBinaryPrinter(byteSink);
+ printer.writeComponentFile(singleLibrary);
+ List<int> data = byteSink.builder.takeBytes();
+ for (Library lib in libraries) {
+ cachedPackageLibraries[lib.fileUri] = data;
+ }
+ ioSink.add(data);
+ }
+ }
+
@override
void acceptLastDelta() {
_generator.accept();
@@ -558,6 +629,17 @@
}
}
+/// A [Sink] that directly writes data into a byte builder.
+class ByteSink implements Sink<List<int>> {
+ final BytesBuilder builder = new BytesBuilder();
+
+ void add(List<int> data) {
+ builder.add(data);
+ }
+
+ void close() {}
+}
+
String _escapePath(String path) {
return path.replaceAll(r'\', r'\\').replaceAll(r' ', r'\ ');
}
@@ -758,10 +840,9 @@
}
}
- compiler ??= new FrontendCompiler(
- output,
- printerFactory: binaryPrinterFactory,
- );
+ compiler ??= new FrontendCompiler(output,
+ printerFactory: binaryPrinterFactory,
+ unsafePackageSerialization: options["unsafe-package-serialization"]);
if (options.rest.isNotEmpty) {
return await compiler.compile(options.rest[0], options,
