Add SkiaPerfPoint and FlutterEngineMetricPoint (#70153)
Unit tests that cover their translations are also added.
diff --git a/dev/benchmarks/metrics_center/lib/src/flutter.dart b/dev/benchmarks/metrics_center/lib/src/flutter.dart
new file mode 100644
index 0000000..9706cf5
--- /dev/null
+++ b/dev/benchmarks/metrics_center/lib/src/flutter.dart
@@ -0,0 +1,22 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:metrics_center/src/common.dart';
+
+/// Convenient class to capture the benchmarks in the Flutter engine repo.
+class FlutterEngineMetricPoint extends MetricPoint {
+ FlutterEngineMetricPoint(
+ String name,
+ double value,
+ String gitRevision, {
+ Map<String, String> moreTags = const <String, String>{},
+ }) : super(
+ value,
+ <String, String>{
+ kNameKey: name,
+ kGithubRepoKey: kFlutterEngineRepo,
+ kGitRevisionKey: gitRevision,
+ }..addAll(moreTags),
+ );
+}
diff --git a/dev/benchmarks/metrics_center/lib/src/github_helper.dart b/dev/benchmarks/metrics_center/lib/src/github_helper.dart
new file mode 100644
index 0000000..f8a8e0f
--- /dev/null
+++ b/dev/benchmarks/metrics_center/lib/src/github_helper.dart
@@ -0,0 +1,39 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file uses Dart 2.12 semantics. This is needed as we can't upgrade
+// the SDK constraint to `>=2.12.0-0` before the deps are ready.
+// @dart=2.12
+
+import 'package:github/github.dart';
+
+/// Singleton class to query some Github info with an in-memory cache.
+class GithubHelper {
+ /// Return the singleton helper.
+ factory GithubHelper() {
+ return _singleton;
+ }
+
+ GithubHelper._internal();
+
+ /// The result is cached in memory so querying the same thing again in the
+ /// same process is fast.
+ ///
+ /// Our unit test requires that calling this method 1000 times for the same
+ /// `githubRepo` and `sha` should be done in 1 second.
+ Future<DateTime> getCommitDateTime(String githubRepo, String sha) async {
+ final String key = '$githubRepo/commit/$sha';
+ if (_commitDateTimeCache[key] == null) {
+ final RepositoryCommit commit = await _github.repositories
+ .getCommit(RepositorySlug.full(githubRepo), sha);
+ _commitDateTimeCache[key] = commit.commit.committer.date;
+ }
+ return _commitDateTimeCache[key]!;
+ }
+
+ static final GithubHelper _singleton = GithubHelper._internal();
+
+ final GitHub _github = GitHub();
+ final Map<String, DateTime> _commitDateTimeCache = <String, DateTime>{};
+}
diff --git a/dev/benchmarks/metrics_center/lib/src/skiaperf.dart b/dev/benchmarks/metrics_center/lib/src/skiaperf.dart
new file mode 100644
index 0000000..e0ad7d9
--- /dev/null
+++ b/dev/benchmarks/metrics_center/lib/src/skiaperf.dart
@@ -0,0 +1,186 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file uses Dart 2.12 semantics. This is needed as we can't upgrade
+// the SDK constraint to `>=2.12.0-0` before the deps are ready.
+// @dart=2.12
+
+import 'package:metrics_center/src/common.dart';
+
+// Skia Perf Format is a JSON file that looks like:
+
+// {
+// "gitHash": "fe4a4029a080bc955e9588d05a6cd9eb490845d4",
+// "key": {
+// "arch": "x86",
+// "gpu": "GTX660",
+// "model": "ShuttleA",
+// "os": "Ubuntu12"
+// },
+// "results": {
+// "ChunkAlloc_PushPop_640_480": {
+// "nonrendering": {
+// "min_ms": 0.01485466666666667,
+// "options": {
+// "source_type": "bench"
+// }
+// }
+// },
+// "DeferredSurfaceCopy_discardable_640_480": {
+// "565": {
+// "min_ms": 2.215988,
+// "options": {
+// "source_type": "bench"
+// }
+// },
+// ...
+
+class SkiaPerfPoint extends MetricPoint {
+ SkiaPerfPoint._(this.githubRepo, this.gitHash, this.testName, this.subResult,
+ double value, this._options, this.jsonUrl)
+ : assert(_options[kGithubRepoKey] == null),
+ assert(_options[kGitRevisionKey] == null),
+ assert(_options[kNameKey] == null),
+ super(
+ value,
+ <String, String>{}
+ ..addAll(_options)
+ ..addAll(<String, String>{
+ kGithubRepoKey: githubRepo,
+ kGitRevisionKey: gitHash,
+ kNameKey: testName,
+ kSubResultKey: subResult,
+ }),
+ ) {
+ assert(tags[kGithubRepoKey] != null);
+ assert(tags[kGitRevisionKey] != null);
+ assert(tags[kNameKey] != null);
+ }
+
+ /// Construct [SkiaPerfPoint] from a well-formed [MetricPoint].
+ ///
+ /// The [MetricPoint] must have [kGithubRepoKey], [kGitRevisionKey],
+ /// [kNameKey] in its tags for this to be successful.
+ ///
+ /// If the [MetricPoint] has a tag 'date', that tag will be removed so Skia
+ /// perf can plot multiple metrics with different date as a single trace.
+ /// Skia perf will use the git revision's date instead of this date tag in
+ /// the time axis.
+ factory SkiaPerfPoint.fromPoint(MetricPoint p) {
+ final String githubRepo = p.tags[kGithubRepoKey]!;
+ final String gitHash = p.tags[kGitRevisionKey]!;
+ final String name = p.tags[kNameKey]!;
+ final String subResult = p.tags[kSubResultKey] ?? kSkiaPerfValueKey;
+
+ final Map<String, String> options = <String, String>{}..addEntries(
+ p.tags.entries.where(
+ (MapEntry<String, dynamic> entry) =>
+ entry.key != kGithubRepoKey &&
+ entry.key != kGitRevisionKey &&
+ entry.key != kNameKey &&
+ entry.key != kSubResultKey &&
+ // https://github.com/google/benchmark automatically generates a
+ // 'date' field. If it's included in options, the Skia perf won't
+ // be able to connect different points in a single trace because
+ // the date is always different.
+ entry.key != 'date',
+ ),
+ );
+
+ return SkiaPerfPoint._(
+ githubRepo, gitHash, name, subResult, p.value, options, null);
+ }
+
+ /// In the format of '<owner>/<name>' such as 'flutter/flutter' or
+ /// 'flutter/engine'.
+ final String githubRepo;
+
+ /// SHA such as 'ad20d368ffa09559754e4b2b5c12951341ca3b2d'
+ final String gitHash;
+
+ /// For Flutter devicelab, this is the task name (e.g.,
+ /// 'flutter_gallery__transition_perf'); for Google benchmark, this is the
+ /// benchmark name (e.g., 'BM_ShellShutdown').
+ ///
+ /// In Skia perf web dashboard, this value can be queried and filtered by
+ /// "test".
+ final String testName;
+
+ /// The name of "subResult" comes from the special treatment of "sub_result"
+ /// in SkiaPerf. If not provided, its value will be set to kSkiaPerfValueKey.
+ ///
+ /// When Google benchmarks are converted to SkiaPerfPoint, this subResult
+ /// could be "cpu_time" or "real_time".
+ ///
+ /// When devicelab benchmarks are converted to SkiaPerfPoint, this subResult
+ /// is often the metric name such as "average_frame_build_time_millis" whereas
+ /// the [testName] is the benchmark or task name such as
+ /// "flutter_gallery__transition_perf".
+ final String subResult;
+
+ /// The url to the Skia perf json file in the Google Cloud Storage bucket.
+ ///
+ /// This can be null if the point has been stored in the bucket yet.
+ final String? jsonUrl;
+
+ Map<String, dynamic> _toSubResultJson() {
+ return <String, dynamic>{
+ subResult: value,
+ kSkiaPerfOptionsKey: _options,
+ };
+ }
+
+ /// Convert a list of SkiaPoints with the same git repo and git revision into
+ /// a single json file in the Skia perf format.
+ ///
+ /// The list must be non-empty.
+ static Map<String, dynamic> toSkiaPerfJson(List<SkiaPerfPoint> points) {
+ assert(points.isNotEmpty);
+ assert(() {
+ for (final SkiaPerfPoint p in points) {
+ if (p.githubRepo != points[0].githubRepo ||
+ p.gitHash != points[0].gitHash) {
+ return false;
+ }
+ }
+ return true;
+ }(), 'All points must have same githubRepo and gitHash');
+
+ final Map<String, dynamic> results = <String, dynamic>{};
+ for (final SkiaPerfPoint p in points) {
+ final Map<String, dynamic> subResultJson = p._toSubResultJson();
+ if (results[p.testName] == null) {
+ results[p.testName] = <String, dynamic>{
+ kSkiaPerfDefaultConfig: subResultJson,
+ };
+ } else {
+ // Flutter currently doesn't support having the same name but different
+ // options/configurations. If this actually happens in the future, we
+ // probably can use different values of config (currently there's only
+ // one kSkiaPerfDefaultConfig) to resolve the conflict.
+ assert(results[p.testName][kSkiaPerfDefaultConfig][kSkiaPerfOptionsKey]
+ .toString() ==
+ subResultJson[kSkiaPerfOptionsKey].toString());
+ assert(
+ results[p.testName][kSkiaPerfDefaultConfig][p.subResult] == null);
+ results[p.testName][kSkiaPerfDefaultConfig][p.subResult] = p.value;
+ }
+ }
+
+ return <String, dynamic>{
+ kSkiaPerfGitHashKey: points[0].gitHash,
+ kSkiaPerfResultsKey: results,
+ };
+ }
+
+ // Equivalent to tags without git repo, git hash, and name because those two
+ // are already stored somewhere else.
+ final Map<String, String> _options;
+}
+
+const String kSkiaPerfGitHashKey = 'gitHash';
+const String kSkiaPerfResultsKey = 'results';
+const String kSkiaPerfValueKey = 'value';
+const String kSkiaPerfOptionsKey = 'options';
+const String kSkiaPerfDefaultConfig = 'default';
diff --git a/dev/benchmarks/metrics_center/test/github_helper_test.dart b/dev/benchmarks/metrics_center/test/github_helper_test.dart
new file mode 100644
index 0000000..3d53248
--- /dev/null
+++ b/dev/benchmarks/metrics_center/test/github_helper_test.dart
@@ -0,0 +1,41 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@Timeout(Duration(seconds: 3600))
+
+import 'package:metrics_center/src/github_helper.dart';
+
+import 'common.dart';
+
+void main() {
+ test('GithubHelper gets correct commit date time', () async {
+ final GithubHelper helper = GithubHelper();
+ expect(
+ await helper.getCommitDateTime(
+ 'flutter/flutter',
+ 'ad20d368ffa09559754e4b2b5c12951341ca3b2d',
+ ),
+ equals(DateTime.parse('2019-12-06 03:33:01.000Z')),
+ );
+ });
+
+ test('GithubHelper is a singleton', () {
+ final GithubHelper helper1 = GithubHelper();
+ final GithubHelper helper2 = GithubHelper();
+ expect(helper1, equals(helper2));
+ });
+
+ test('GithubHelper can query the same commit 1000 times within 1 second',
+ () async {
+ final DateTime start = DateTime.now();
+ for (int i = 0; i < 1000; i += 1) {
+ await GithubHelper().getCommitDateTime(
+ 'flutter/flutter',
+ 'ad20d368ffa09559754e4b2b5c12951341ca3b2d',
+ );
+ }
+ final Duration duration = DateTime.now().difference(start);
+ expect(duration, lessThan(const Duration(seconds: 1)));
+ });
+}
diff --git a/dev/benchmarks/metrics_center/test/google_benchmark_test.dart b/dev/benchmarks/metrics_center/test/google_benchmark_test.dart
index 1661204..629b5a0 100644
--- a/dev/benchmarks/metrics_center/test/google_benchmark_test.dart
+++ b/dev/benchmarks/metrics_center/test/google_benchmark_test.dart
@@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-// @dart = 2.8
-
import 'package:metrics_center/src/common.dart';
import 'package:metrics_center/google_benchmark.dart';
diff --git a/dev/benchmarks/metrics_center/test/skiaperf_test.dart b/dev/benchmarks/metrics_center/test/skiaperf_test.dart
new file mode 100644
index 0000000..2547cbb
--- /dev/null
+++ b/dev/benchmarks/metrics_center/test/skiaperf_test.dart
@@ -0,0 +1,265 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@Timeout(Duration(seconds: 3600))
+
+import 'dart:convert';
+
+import 'package:metrics_center/src/common.dart';
+import 'package:metrics_center/src/flutter.dart';
+import 'package:metrics_center/src/skiaperf.dart';
+
+import 'common.dart';
+
+void main() {
+ const double kValue1 = 1.0;
+ const double kValue2 = 2.0;
+
+ const String kFrameworkRevision1 = '9011cece2595447eea5dd91adaa241c1c9ef9a33';
+ const String kTaskName = 'analyzer_benchmark';
+ const String kMetric1 = 'flutter_repo_batch_maximum';
+ const String kMetric2 = 'flutter_repo_watch_maximum';
+
+ final MetricPoint cocoonPointRev1Metric1 = MetricPoint(
+ kValue1,
+ const <String, String>{
+ kGithubRepoKey: kFlutterFrameworkRepo,
+ kGitRevisionKey: kFrameworkRevision1,
+ kNameKey: kTaskName,
+ kSubResultKey: kMetric1,
+ kUnitKey: 's',
+ },
+ );
+
+ final MetricPoint cocoonPointRev1Metric2 = MetricPoint(
+ kValue2,
+ const <String, String>{
+ kGithubRepoKey: kFlutterFrameworkRepo,
+ kGitRevisionKey: kFrameworkRevision1,
+ kNameKey: kTaskName,
+ kSubResultKey: kMetric2,
+ kUnitKey: 's',
+ },
+ );
+
+ final MetricPoint cocoonPointBetaRev1Metric1 = MetricPoint(
+ kValue1,
+ const <String, String>{
+ kGithubRepoKey: kFlutterFrameworkRepo,
+ kGitRevisionKey: kFrameworkRevision1,
+ kNameKey: 'beta/$kTaskName',
+ kSubResultKey: kMetric1,
+ kUnitKey: 's',
+ 'branch': 'beta',
+ },
+ );
+
+ final MetricPoint cocoonPointBetaRev1Metric1BadBranch = MetricPoint(
+ kValue1,
+ const <String, String>{
+ kGithubRepoKey: kFlutterFrameworkRepo,
+ kGitRevisionKey: kFrameworkRevision1,
+ kNameKey: kTaskName,
+ kSubResultKey: kMetric1,
+ kUnitKey: 's',
+
+ // If we only add this 'branch' tag without changing the test or sub-result name, an exception
+ // would be thrown as Skia Perf currently only supports the same set of tags for a pair of
+ // kNameKey and kSubResultKey values. So to support branches, one also has to add the branch
+ // name to the test name.
+ 'branch': 'beta',
+ },
+ );
+
+ const String engineMetricName = 'BM_PaintRecordInit';
+ const String engineRevision = 'ca799fa8b2254d09664b78ee80c43b434788d112';
+ const double engineValue1 = 101;
+ const double engineValue2 = 102;
+
+ final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint(
+ engineMetricName,
+ engineValue1,
+ engineRevision,
+ moreTags: const <String, String>{
+ kSubResultKey: 'cpu_time',
+ kUnitKey: 'ns',
+ 'date': '2019-12-17 15:14:14',
+ 'num_cpus': '56',
+ 'mhz_per_cpu': '2594',
+ 'cpu_scaling_enabled': 'true',
+ 'library_build_type': 'release',
+ },
+ );
+
+ final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint(
+ engineMetricName,
+ engineValue2,
+ engineRevision,
+ moreTags: const <String, String>{
+ kSubResultKey: 'real_time',
+ kUnitKey: 'ns',
+ 'date': '2019-12-17 15:14:14',
+ 'num_cpus': '56',
+ 'mhz_per_cpu': '2594',
+ 'cpu_scaling_enabled': 'true',
+ 'library_build_type': 'release',
+ },
+ );
+
+ test('Throw if invalid points are converted to SkiaPoint', () {
+ final MetricPoint noGithubRepoPoint = MetricPoint(
+ kValue1,
+ const <String, String>{
+ kGitRevisionKey: kFrameworkRevision1,
+ kNameKey: kTaskName,
+ },
+ );
+
+ final MetricPoint noGitRevisionPoint = MetricPoint(
+ kValue1,
+ const <String, String>{
+ kGithubRepoKey: kFlutterFrameworkRepo,
+ kNameKey: kTaskName,
+ },
+ );
+
+ final MetricPoint noTestNamePoint = MetricPoint(
+ kValue1,
+ const <String, String>{
+ kGithubRepoKey: kFlutterFrameworkRepo,
+ kGitRevisionKey: kFrameworkRevision1,
+ },
+ );
+
+ expect(() => SkiaPerfPoint.fromPoint(noGithubRepoPoint), throwsA(anything));
+ expect(
+ () => SkiaPerfPoint.fromPoint(noGitRevisionPoint), throwsA(anything));
+ expect(() => SkiaPerfPoint.fromPoint(noTestNamePoint), throwsA(anything));
+ });
+
+ test('Correctly convert a metric point from cocoon to SkiaPoint', () {
+ final SkiaPerfPoint skiaPoint1 =
+ SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
+ expect(skiaPoint1, isNotNull);
+ expect(skiaPoint1.testName, equals(kTaskName));
+ expect(skiaPoint1.subResult, equals(kMetric1));
+ expect(skiaPoint1.value, equals(cocoonPointRev1Metric1.value));
+ expect(skiaPoint1.jsonUrl, isNull); // Not inserted yet
+ });
+
+ test('Cocoon points correctly encode into Skia perf json format', () {
+ final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
+ final SkiaPerfPoint p2 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric2);
+ final SkiaPerfPoint p3 =
+ SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1);
+
+ const JsonEncoder encoder = JsonEncoder.withIndent(' ');
+
+ expect(
+ encoder
+ .convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[p1, p2, p3])),
+ equals('''
+{
+ "gitHash": "9011cece2595447eea5dd91adaa241c1c9ef9a33",
+ "results": {
+ "analyzer_benchmark": {
+ "default": {
+ "flutter_repo_batch_maximum": 1.0,
+ "options": {
+ "unit": "s"
+ },
+ "flutter_repo_watch_maximum": 2.0
+ }
+ },
+ "beta/analyzer_benchmark": {
+ "default": {
+ "flutter_repo_batch_maximum": 1.0,
+ "options": {
+ "branch": "beta",
+ "unit": "s"
+ }
+ }
+ }
+ }
+}'''));
+ });
+
+ test('Engine metric points correctly encode into Skia perf json format', () {
+ const JsonEncoder encoder = JsonEncoder.withIndent(' ');
+ expect(
+ encoder.convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[
+ SkiaPerfPoint.fromPoint(enginePoint1),
+ SkiaPerfPoint.fromPoint(enginePoint2),
+ ])),
+ equals(
+ '''
+{
+ "gitHash": "ca799fa8b2254d09664b78ee80c43b434788d112",
+ "results": {
+ "BM_PaintRecordInit": {
+ "default": {
+ "cpu_time": 101.0,
+ "options": {
+ "cpu_scaling_enabled": "true",
+ "library_build_type": "release",
+ "mhz_per_cpu": "2594",
+ "num_cpus": "56",
+ "unit": "ns"
+ },
+ "real_time": 102.0
+ }
+ }
+ }
+}''',
+ ),
+ );
+ });
+
+ test(
+ 'Throw if engine points with the same test name but different options are converted to '
+ 'Skia perf points', () {
+ final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint(
+ 'BM_PaintRecordInit',
+ 101,
+ 'ca799fa8b2254d09664b78ee80c43b434788d112',
+ moreTags: const <String, String>{
+ kSubResultKey: 'cpu_time',
+ kUnitKey: 'ns',
+ 'cpu_scaling_enabled': 'true',
+ },
+ );
+ final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint(
+ 'BM_PaintRecordInit',
+ 102,
+ 'ca799fa8b2254d09664b78ee80c43b434788d112',
+ moreTags: const <String, String>{
+ kSubResultKey: 'real_time',
+ kUnitKey: 'ns',
+ 'cpu_scaling_enabled': 'false',
+ },
+ );
+
+ const JsonEncoder encoder = JsonEncoder.withIndent(' ');
+ expect(
+ () => encoder.convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[
+ SkiaPerfPoint.fromPoint(enginePoint1),
+ SkiaPerfPoint.fromPoint(enginePoint2),
+ ])),
+ throwsA(anything),
+ );
+ });
+
+ test(
+ 'Throw if two Cocoon metric points with the same name and subResult keys '
+ 'but different options are converted to Skia perf points', () {
+ final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
+ final SkiaPerfPoint p2 =
+ SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1BadBranch);
+
+ expect(
+ () => SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[p1, p2]),
+ throwsA(anything),
+ );
+ });
+}
