commit 06adfb8d6b9686ba42ccf1db0202ef54f591aa06
author Nate Biggs <natebiggs@google.com> Fri Jan 12 17:11:51 2024 +0000
committer Commit Queue <dart-scoped@luci-project-accounts.iam.gserviceaccount.com> Fri Jan 12 17:11:51 2024 +0000
tree 48c13254a085dfa3ee51f57ec26ef5b72bd0ca39
parent c46b63d2c79eb996fbea3c1ef3a44c4692c48000

[dart2js] Fix resource usage of binary to json converter tool.

Improves binary -> json conversion from >900s to ~52s for a large program.

This change contains 2 major fixes:
1) Make element name deduping be O(1) rather than O(n) by using a hash map. I didn't calculate how many duplicated names there were but this change improved the runtime of the conversion process by ~10x.

2) Use chunked string writing for the json encoding/file writing process. By doing chunked conversion with buffered writes we avoid having the entire JSON blob in memory and fewer midsized write operations end up being faster. Without this the program took ~87s for the same large program.

Change-Id: I2d3cddbbb96895b68086fce2256937d567beff11
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/345940
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Commit-Queue: Nate Biggs <natebiggs@google.com>

pkg/dart2js_info/bin/src/to_json.dart

diff --git a/pkg/dart2js_info/bin/src/to_json.dart b/pkg/dart2js_info/bin/src/to_json.dart
index 5a6c806..7aca9c8 100644
--- a/pkg/dart2js_info/bin/src/to_json.dart
+++ b/pkg/dart2js_info/bin/src/to_json.dart
@@ -51,7 +51,33 @@
     var json = AllInfoJsonCodec(isBackwardCompatible: isBackwardCompatible)
         .encode(info);
     String outputFilename = args['out'] ?? '$filename.json';
-    File(outputFilename)
-        .writeAsStringSync(const JsonEncoder.withIndent("  ").convert(json));
+    final sink = File(outputFilename).openWrite();
+    final converterSink = const JsonEncoder.withIndent("  ")
+        .startChunkedConversion(_BufferedStringOutputSink(sink));
+    converterSink.add(json);
+    converterSink.close();
+    await sink.close();
+  }
+}
+
+class _BufferedStringOutputSink implements Sink<String> {
+  StringBuffer buffer = StringBuffer();
+  final StringSink outputSink;
+  static const int _maxLength = 1024 * 1024 * 500;
+
+  _BufferedStringOutputSink(this.outputSink);
+
+  @override
+  void add(String data) {
+    buffer.write(data);
+    if (buffer.length > _maxLength) {
+      outputSink.write(buffer.toString());
+      buffer.clear();
+    }
+  }
+
+  @override
+  void close() {
+    outputSink.write(buffer.toString());
   }
 }

pkg/dart2js_info/lib/json_info_codec.dart

diff --git a/pkg/dart2js_info/lib/json_info_codec.dart b/pkg/dart2js_info/lib/json_info_codec.dart
index 19f4b2d..6624e8f 100644
--- a/pkg/dart2js_info/lib/json_info_codec.dart
+++ b/pkg/dart2js_info/lib/json_info_codec.dart
@@ -346,7 +346,7 @@
   final bool isBackwardCompatible;
 
   final Map<Info, Id> ids = HashMap<Info, Id>();
-  final Set<String> usedIds = <String>{};
+  final Map<String, int> idCounter = <String, int>{};
 
   AllInfoToJsonConverter({this.isBackwardCompatible = false});
 
@@ -373,14 +373,10 @@
       name = longName(info, useLibraryUri: true, forId: true);
     }
 
-    Id id = Id(info.kind, name);
     // longName isn't guaranteed to create unique serializedIds for some info
     // constructs (such as closures), so we disambiguate here.
-    int count = 0;
-    while (!usedIds.add(id.serializedId)) {
-      id = Id(info.kind, '$name%${count++}');
-    }
-
+    final count = idCounter.update(name, (v) => v + 1, ifAbsent: () => 0);
+    final id = Id(info.kind, count == 0 ? name : '$name%$count');
     return ids[info] = id;
   }