pkg/front_end/tool/compare_revisions_tool.dart - sdk - Git at Google

 // Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.

 import "dart:io";

 import "benchmarker.dart" as benchmarker;

 const String compileDartRelativePath = "pkg/front_end/tool/_fasta/compile.dart";

 const int iterations = 3;

 String? target;
 String? changingWorkingDir;
 String? sdkPath;
 String? snapshotsPath;

 void main(List<String> args) {
   if (args.contains("--help")) return _help();
   bool filter = false;
   bool raw = false;
   List<String> examines = [];
   List<String> extraVmArguments = [];
   for (String arg in args) {
     if (arg.startsWith("--target=")) {
       target = arg.substring("--target=".length);
     } else if (arg.startsWith("--changingWorkingDir=")) {
       changingWorkingDir = arg.substring("--changingWorkingDir=".length);
     } else if (arg.startsWith("--sdkPath=")) {
       sdkPath = arg.substring("--sdkPath=".length);
     } else if (arg.startsWith("--snapshotsPath=")) {
       snapshotsPath = arg.substring("--snapshotsPath=".length);
     } else if (arg == "--filter") {
       filter = true;
     } else if (arg == "--raw") {
       raw = true;
     } else if (arg.startsWith("--examine=")) {
       examines.addAll(arg.substring("--examine=".length).split(","));
     } else if (arg.startsWith("--extraVmArguments=")) {
       // E.g. "--old_gen_growth_rate=1000" in an attempt to even out GC stuff.
       extraVmArguments.add(arg.substring("--extraVmArguments=".length));
     } else {
       throw "Unknown argument: $arg";
     }
   }

   if (target == null) throw "Specify --target";
   if (changingWorkingDir == null) throw "Specify --changingWorkingDir";
   if (snapshotsPath == null) throw "Specify --snapshotsPath";
   if (sdkPath == null) throw "Specify --sdkPath";
   if (!new File("${sdkPath}bin/dart").existsSync()) {
     throw "--sdkPath doesn't contain bin/dart";
   }
   if (!new File("${sdkPath}bin/dartaotruntime").existsSync()) {
     throw "--sdkPath doesn't contain bin/dartaotruntime";
   }
   if (examines.isEmpty) {
     throw "Specify one or more commits to examine via --examine=";
   }

   for (String examine in examines) {
     if (examine.trim() == "") continue;
     _examine(examine, filter, raw, extraVmArguments);
   }
 }

 void _help() {
   print("CFE revision benchmarking tool");
   print("");
   print("Specify target, i.e. what we'll compile when benchmarking via");
   print("--target=<dart file>");
   print("Specify a git checkout that this script can manage");
   print("(and delete untracked files etc in) via");
   print("--changingWorkingDir=<checkout dir>");
   print("Specify the sdk to use via");
   print("--sdkPath=<path>");
   print("Specify where to save the snapshots via");
   print("--snapshotsPath=<path>");
   print("Specify which commit(s) to examine with");
   print("--examine=<revision>");
   print("(specify more either with more --examine= arguments,");
   print("or by comma-separation)");
   print("");
   print("Control output as needed with");
   print("--filter");
   print("and");
   print("--raw");
   print("");
   print("Specify extra arguments to pass to the VM with");
   print("--extraVmArguments=<whatever>");
   print("E.g. `--extraVmArguments=--old_gen_growth_rate=1000`");
   print("");
   print("Example run:");
   print("");
   print(r"out/ReleaseX64/dart-sdk/bin/dart \");
   print(r"  pkg/front_end/tool/compare_revisions_tool.dart \");
   print(r"  --target=pkg/front_end/tool/_fasta/compile.dart \");
   print(r"  --changingWorkingDir=/tmp/tmp-playing-with-git/sdk/ \");
   print(r"  --sdkPath=67e9580b042/dart-sdk/ \");
   print(r"  --snapshotsPath=67e9580b042 \");
   print(r"  --examine=e7deece1fb2,529e016a0a7,fd02fec0fc4 \");
 }

 void _compileRevision(String gitCommit, {Stopwatch? stopwatch}) {
   if (new File("$snapshotsPath/platform.dill.$gitCommit").existsSync() &&
       new File("$snapshotsPath/compile.aot.$gitCommit").existsSync()) {
     return;
   }
   stopwatch ??= new Stopwatch()..start();

   // Clean up the git checkout.
   ProcessResult processResult = Process.runSync("git", ["reset", "--hard"],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed resetting hard for $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }
   processResult = Process.runSync("git", ["clean", "-d", "-f"],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed cleaning for $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }

   // Checkout at the specific revision.
   processResult = Process.runSync("git", ["checkout", gitCommit],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed checking out $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }
   print("Done running git checkout for $gitCommit after "
       "${stopwatch.elapsed.inSeconds} seconds.");

   // Clean up the git checkout.
   processResult = Process.runSync("git", ["reset", "--hard"],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed resetting hard for $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }
   processResult = Process.runSync("git", ["clean", "-d", "-f"],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed cleaning for $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }

   // Run `gclient sync`.
   processResult = Process.runSync("gclient", ["sync", "-D"],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed gclient sync at $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }
   print("Done running `gclient sync` for $gitCommit "
       "after ${stopwatch.elapsed.inSeconds} seconds.");

   // Build the platform and copy it so we have it.
   processResult = Process.runSync(
       "python3", ["tools/build.py", "-ax64", "-mrelease", "vm_platform"],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed compile vm platform at $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }
   new File("$changingWorkingDir/out/ReleaseX64/vm_platform_strong.dill")
       .copySync("$snapshotsPath/platform.dill.$gitCommit");
   print("Done building the platform for $gitCommit "
       "after ${stopwatch.elapsed.inSeconds} seconds.");

   // Compile the AOT snapshot.
   processResult = Process.runSync("${sdkPath}bin/dart", [
     "compile",
     "aot-snapshot",
     "$changingWorkingDir/$compileDartRelativePath",
     "-o",
     "$snapshotsPath/compile.aot.$gitCommit"
   ]);
   if (processResult.exitCode != 0) {
     throw "Failed compile aot-snapshot at $gitCommit:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }
   print("Compiled for $gitCommit "
       "after ${stopwatch.elapsed.inSeconds} seconds.");
 }

 void _examine(
     String revision, bool filter, bool raw, List<String> extraVmArguments) {
   ProcessResult processResult = Process.runSync(
       "git", ["log", "$revision^^...$revision", "--pretty=format:%h"],
       workingDirectory: changingWorkingDir);
   if (processResult.exitCode != 0) {
     throw "Failed to get log:\n"
         "stderr: ${processResult.stderr}\n"
         "stdout: ${processResult.stdout}";
   }

   List<String> revisions = processResult.stdout.split("\n").reversed.toList();
   if (revisions.length != 2) throw "Expected 2 revisions but got $revisions";
   if (revisions.last != revision) {
     throw "Expected $revision but got ${revisions.last}";
   }
   String prevRevision = revisions.first;

   print("Creating AOT-snapshots if needed.");
   _compileRevision(prevRevision);
   _compileRevision(revision);

   print("Will now examine $prevRevision -> $revision.");

   print("Running with verbose GC.");
   benchmarker.GCInfo gcPrev = _runVerboseGc(prevRevision, extraVmArguments);
   benchmarker.GCInfo gcCurrent = _runVerboseGc(revision, extraVmArguments);
   benchmarker.printGcDiff(gcPrev, gcCurrent);

   print("Running $iterations iterations for $prevRevision.");
   List<Map<String, num>> benchmarkDataFrom = [];
   _run(iterations, prevRevision, extraVmArguments, benchmarkDataFrom);

   print("Running $iterations iterations for $revision.");
   List<Map<String, num>> benchmarkDataTo = [];
   _run(iterations, revision, extraVmArguments, benchmarkDataTo);

   if (filter) {
     // Filter to only "instructions:u".
     benchmarkDataFrom = _filterToInstructions(benchmarkDataFrom);
     benchmarkDataTo = _filterToInstructions(benchmarkDataTo);
   }
   if (raw) {
     print("From: $benchmarkDataFrom");
     print("To: $benchmarkDataTo");
   }

   print("Examine: $prevRevision -> $revision:");
   if (!benchmarker.compare(benchmarkDataFrom, benchmarkDataTo)) {
     print("No change.");
   }
 }

 void _run(int iterations, String gitCommit, List<String> extraVmArguments,
     List<Map<String, num>> output) {
   for (int i = 0; i < iterations; i++) {
     try {
       output.add(benchmarker.benchmark(
           "$snapshotsPath/compile.aot.$gitCommit",
           extraVmArguments,
           [
             "--platform=$snapshotsPath/platform.dill.$gitCommit",
             target!,
           ],
           aotRuntime: "${sdkPath}bin/dartaotruntime"));
     } catch (e) {
       throw "Failed to run benchmark at $gitCommit";
     }
   }
 }

 benchmarker.GCInfo _runVerboseGc(
     String gitCommit, List<String> extraVmArguments) {
   ProcessResult processResult =
       Process.runSync("${sdkPath}bin/dartaotruntime", [
     "--deterministic",
     "--verbose-gc",
     ...extraVmArguments,
     "$snapshotsPath/compile.aot.$gitCommit",
     "--platform=$snapshotsPath/platform.dill.$gitCommit",
     target!
   ]);

   if (processResult.exitCode != 0) {
     throw "Run failed for $gitCommit with exit code "
         "${processResult.exitCode}.\n"
         "stdout:\n${processResult.stdout}\n\n"
         "stderr:\n${processResult.stderr}\n\n";
   }

   return benchmarker.parseVerboseGcOutput(processResult);
 }

 List<Map<String, num>> _filterToInstructions(List<Map<String, num>> input,
     {List<num>? extractedNumbers}) {
   List<Map<String, num>> result = [];
   for (Map<String, num> map in input) {
     num? instructionsValue = map["instructions:u"];
     if (instructionsValue != null) {
       result.add({"instructions:u": instructionsValue});
       extractedNumbers?.add(instructionsValue);
     }
   }
   return result;
 }
	// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
	// for details. All rights reserved. Use of this source code is governed by a
	// BSD-style license that can be found in the LICENSE file.

	import "dart:io";

	import "benchmarker.dart" as benchmarker;

	const String compileDartRelativePath = "pkg/front_end/tool/_fasta/compile.dart";

	const int iterations = 3;

	String? target;
	String? changingWorkingDir;
	String? sdkPath;
	String? snapshotsPath;

	void main(List<String> args) {
	if (args.contains("--help")) return _help();
	bool filter = false;
	bool raw = false;
	List<String> examines = [];
	List<String> extraVmArguments = [];
	for (String arg in args) {
	if (arg.startsWith("--target=")) {
	target = arg.substring("--target=".length);
	} else if (arg.startsWith("--changingWorkingDir=")) {
	changingWorkingDir = arg.substring("--changingWorkingDir=".length);
	} else if (arg.startsWith("--sdkPath=")) {
	sdkPath = arg.substring("--sdkPath=".length);
	} else if (arg.startsWith("--snapshotsPath=")) {
	snapshotsPath = arg.substring("--snapshotsPath=".length);
	} else if (arg == "--filter") {
	filter = true;
	} else if (arg == "--raw") {
	raw = true;
	} else if (arg.startsWith("--examine=")) {
	examines.addAll(arg.substring("--examine=".length).split(","));
	} else if (arg.startsWith("--extraVmArguments=")) {
	// E.g. "--old_gen_growth_rate=1000" in an attempt to even out GC stuff.
	extraVmArguments.add(arg.substring("--extraVmArguments=".length));
	} else {
	throw "Unknown argument: $arg";
	}
	}

	if (target == null) throw "Specify --target";
	if (changingWorkingDir == null) throw "Specify --changingWorkingDir";
	if (snapshotsPath == null) throw "Specify --snapshotsPath";
	if (sdkPath == null) throw "Specify --sdkPath";
	if (!new File("${sdkPath}bin/dart").existsSync()) {
	throw "--sdkPath doesn't contain bin/dart";
	}
	if (!new File("${sdkPath}bin/dartaotruntime").existsSync()) {
	throw "--sdkPath doesn't contain bin/dartaotruntime";
	}
	if (examines.isEmpty) {
	throw "Specify one or more commits to examine via --examine=";
	}

	for (String examine in examines) {
	if (examine.trim() == "") continue;
	_examine(examine, filter, raw, extraVmArguments);
	}
	}

	void _help() {
	print("CFE revision benchmarking tool");
	print("");
	print("Specify target, i.e. what we'll compile when benchmarking via");
	print("--target=<dart file>");
	print("Specify a git checkout that this script can manage");
	print("(and delete untracked files etc in) via");
	print("--changingWorkingDir=<checkout dir>");
	print("Specify the sdk to use via");
	print("--sdkPath=<path>");
	print("Specify where to save the snapshots via");
	print("--snapshotsPath=<path>");
	print("Specify which commit(s) to examine with");
	print("--examine=<revision>");
	print("(specify more either with more --examine= arguments,");
	print("or by comma-separation)");
	print("");
	print("Control output as needed with");
	print("--filter");
	print("and");
	print("--raw");
	print("");
	print("Specify extra arguments to pass to the VM with");
	print("--extraVmArguments=<whatever>");
	print("E.g. `--extraVmArguments=--old_gen_growth_rate=1000`");
	print("");
	print("Example run:");
	print("");
	print(r"out/ReleaseX64/dart-sdk/bin/dart \");
	print(r" pkg/front_end/tool/compare_revisions_tool.dart \");
	print(r" --target=pkg/front_end/tool/_fasta/compile.dart \");
	print(r" --changingWorkingDir=/tmp/tmp-playing-with-git/sdk/ \");
	print(r" --sdkPath=67e9580b042/dart-sdk/ \");
	print(r" --snapshotsPath=67e9580b042 \");
	print(r" --examine=e7deece1fb2,529e016a0a7,fd02fec0fc4 \");
	}

	void _compileRevision(String gitCommit, {Stopwatch? stopwatch}) {
	if (new File("$snapshotsPath/platform.dill.$gitCommit").existsSync() &&
	new File("$snapshotsPath/compile.aot.$gitCommit").existsSync()) {
	return;
	}
	stopwatch ??= new Stopwatch()..start();

	// Clean up the git checkout.
	ProcessResult processResult = Process.runSync("git", ["reset", "--hard"],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed resetting hard for $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}
	processResult = Process.runSync("git", ["clean", "-d", "-f"],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed cleaning for $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}

	// Checkout at the specific revision.
	processResult = Process.runSync("git", ["checkout", gitCommit],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed checking out $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}
	print("Done running git checkout for $gitCommit after "
	"${stopwatch.elapsed.inSeconds} seconds.");

	// Clean up the git checkout.
	processResult = Process.runSync("git", ["reset", "--hard"],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed resetting hard for $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}
	processResult = Process.runSync("git", ["clean", "-d", "-f"],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed cleaning for $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}

	// Run `gclient sync`.
	processResult = Process.runSync("gclient", ["sync", "-D"],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed gclient sync at $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}
	print("Done running `gclient sync` for $gitCommit "
	"after ${stopwatch.elapsed.inSeconds} seconds.");

	// Build the platform and copy it so we have it.
	processResult = Process.runSync(
	"python3", ["tools/build.py", "-ax64", "-mrelease", "vm_platform"],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed compile vm platform at $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}
	new File("$changingWorkingDir/out/ReleaseX64/vm_platform_strong.dill")
	.copySync("$snapshotsPath/platform.dill.$gitCommit");
	print("Done building the platform for $gitCommit "
	"after ${stopwatch.elapsed.inSeconds} seconds.");

	// Compile the AOT snapshot.
	processResult = Process.runSync("${sdkPath}bin/dart", [
	"compile",
	"aot-snapshot",
	"$changingWorkingDir/$compileDartRelativePath",
	"-o",
	"$snapshotsPath/compile.aot.$gitCommit"
	]);
	if (processResult.exitCode != 0) {
	throw "Failed compile aot-snapshot at $gitCommit:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}
	print("Compiled for $gitCommit "
	"after ${stopwatch.elapsed.inSeconds} seconds.");
	}

	void _examine(
	String revision, bool filter, bool raw, List<String> extraVmArguments) {
	ProcessResult processResult = Process.runSync(
	"git", ["log", "$revision^^...$revision", "--pretty=format:%h"],
	workingDirectory: changingWorkingDir);
	if (processResult.exitCode != 0) {
	throw "Failed to get log:\n"
	"stderr: ${processResult.stderr}\n"
	"stdout: ${processResult.stdout}";
	}

	List<String> revisions = processResult.stdout.split("\n").reversed.toList();
	if (revisions.length != 2) throw "Expected 2 revisions but got $revisions";
	if (revisions.last != revision) {
	throw "Expected $revision but got ${revisions.last}";
	}
	String prevRevision = revisions.first;

	print("Creating AOT-snapshots if needed.");
	_compileRevision(prevRevision);
	_compileRevision(revision);

	print("Will now examine $prevRevision -> $revision.");

	print("Running with verbose GC.");
	benchmarker.GCInfo gcPrev = _runVerboseGc(prevRevision, extraVmArguments);
	benchmarker.GCInfo gcCurrent = _runVerboseGc(revision, extraVmArguments);
	benchmarker.printGcDiff(gcPrev, gcCurrent);

	print("Running $iterations iterations for $prevRevision.");
	List<Map<String, num>> benchmarkDataFrom = [];
	_run(iterations, prevRevision, extraVmArguments, benchmarkDataFrom);

	print("Running $iterations iterations for $revision.");
	List<Map<String, num>> benchmarkDataTo = [];
	_run(iterations, revision, extraVmArguments, benchmarkDataTo);

	if (filter) {
	// Filter to only "instructions:u".
	benchmarkDataFrom = _filterToInstructions(benchmarkDataFrom);
	benchmarkDataTo = _filterToInstructions(benchmarkDataTo);
	}
	if (raw) {
	print("From: $benchmarkDataFrom");
	print("To: $benchmarkDataTo");
	}

	print("Examine: $prevRevision -> $revision:");
	if (!benchmarker.compare(benchmarkDataFrom, benchmarkDataTo)) {
	print("No change.");
	}
	}

	void _run(int iterations, String gitCommit, List<String> extraVmArguments,
	List<Map<String, num>> output) {
	for (int i = 0; i < iterations; i++) {
	try {
	output.add(benchmarker.benchmark(
	"$snapshotsPath/compile.aot.$gitCommit",
	extraVmArguments,
	[
	"--platform=$snapshotsPath/platform.dill.$gitCommit",
	target!,
	],
	aotRuntime: "${sdkPath}bin/dartaotruntime"));
	} catch (e) {
	throw "Failed to run benchmark at $gitCommit";
	}
	}
	}

	benchmarker.GCInfo _runVerboseGc(
	String gitCommit, List<String> extraVmArguments) {
	ProcessResult processResult =
	Process.runSync("${sdkPath}bin/dartaotruntime", [
	"--deterministic",
	"--verbose-gc",
	...extraVmArguments,
	"$snapshotsPath/compile.aot.$gitCommit",
	"--platform=$snapshotsPath/platform.dill.$gitCommit",
	target!
	]);

	if (processResult.exitCode != 0) {
	throw "Run failed for $gitCommit with exit code "
	"${processResult.exitCode}.\n"
	"stdout:\n${processResult.stdout}\n\n"
	"stderr:\n${processResult.stderr}\n\n";
	}

	return benchmarker.parseVerboseGcOutput(processResult);
	}

	List<Map<String, num>> _filterToInstructions(List<Map<String, num>> input,
	{List<num>? extractedNumbers}) {
	List<Map<String, num>> result = [];
	for (Map<String, num> map in input) {
	num? instructionsValue = map["instructions:u"];
	if (instructionsValue != null) {
	result.add({"instructions:u": instructionsValue});
	extractedNumbers?.add(instructionsValue);
	}
	}
	return result;
	}