blob: c0ac7b407800f1a13b8bb5f2b6e0994b6be4f452 [file] [log] [blame]
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// @dart=2.9
import "dart:io";
import "../test/simple_stats.dart";
void usage([String extraMessage]) {
print("Usage:");
print("On Linux via bash you can do something like");
print("dart pkg/front_end/tool/stat_on_dash_v.dart \ "
" now_run_{1..10}.data then_run_{1..10}.data");
if (extraMessage != null) {
print("");
print("Notice:");
print(extraMessage);
}
exit(1);
}
main(List<String> args) {
if (args.length < 4) {
usage("Requires more input.");
}
// Maps from "part" (or "category" or whatever) =>
// (map from file group => list of runtimes)
Map<String, Map<String, List<int>>> data = {};
Set<String> allGroups = {};
for (String file in args) {
File f = new File(file);
if (!f.existsSync()) usage("$file doesn't exist.");
String groupId = replaceNumbers(file);
allGroups.add(groupId);
String fileContent = f.readAsStringSync();
List<String> fileLines = fileContent.split("\n");
Set<String> partsSeen = {};
for (String line in fileLines) {
if (!isTimePrependedLine(line)) continue;
String trimmedLine = line.substring(16).trim();
String part = replaceNumbers(trimmedLine);
if (!partsSeen.add(part)) {
int seen = 2;
while (true) {
String newPartName = "$part ($seen)";
if (partsSeen.add(newPartName)) {
part = newPartName;
break;
}
seen++;
}
}
int microSeconds = findMs(trimmedLine, inMs: true);
Map<String, List<int>> groupToTime = data[part] ??= {};
List<int> times = groupToTime[groupId] ??= [];
times.add(microSeconds);
}
}
if (allGroups.length < 2) {
assert(allGroups.length == 1);
usage("Found only 1 group. At least two are required.");
}
Map<String, double> combinedChange = {};
bool printedAnything = false;
for (String part in data.keys) {
Map<String, List<int>> partData = data[part];
List<int> prevRuntimes;
String prevGroup;
bool printed = false;
for (String group in allGroups) {
List<int> runtimes = partData[group];
if (runtimes == null) {
// Fake it to be a small list of 0s.
runtimes = new List<int>.filled(5, 0);
if (!printed) {
printed = true;
print("$part:");
}
print("Notice: faking data for $group");
}
if (prevRuntimes != null) {
TTestResult result = SimpleTTestStat.ttest(runtimes, prevRuntimes);
if (result.significant) {
if (!printed) {
printed = true;
print("$part:");
}
print("$prevGroup => $group: $result");
print("$group: $runtimes");
print("$prevGroup: $prevRuntimes");
combinedChange["$prevGroup => $group"] ??= 0;
double leastConfidentChange;
if (result.diff < 0) {
leastConfidentChange = result.diff + result.confidence;
} else {
leastConfidentChange = result.diff - result.confidence;
}
combinedChange["$prevGroup => $group"] += leastConfidentChange;
}
}
prevRuntimes = runtimes;
prevGroup = group;
}
if (printed) {
print("---");
printedAnything = true;
}
}
if (printedAnything) {
for (String part in combinedChange.keys) {
print("Combined least change for $part: "
"${combinedChange[part].toStringAsFixed(2)} ms.");
}
} else {
print("Nothing significant found.");
}
}
/// Returns ms or µs or throws if ms not found.
int findMs(String s, {bool inMs: true}) {
// Find " in " followed by numbers possibly followed by (a dot and more
// numbers) followed by "ms"; e.g. " in 42.3ms"
// This is O(n^2) but it doesn't matter.
for (int i = 0; i < s.length; i++) {
int j = 0;
if (s.codeUnitAt(i + j++) != $SPACE) continue;
if (s.codeUnitAt(i + j++) != $i) continue;
if (s.codeUnitAt(i + j++) != $n) continue;
if (s.codeUnitAt(i + j++) != $SPACE) continue;
int numberStartsAt = i + j;
if (!isNumber(s.codeUnitAt(i + j++))) continue;
while (isNumber(s.codeUnitAt(i + j))) {
j++;
}
// We've seen " is 0+" => we should now either have "ms" or a dot,
// more numbers followed by "ms".
if (s.codeUnitAt(i + j) == $m) {
j++;
if (s.codeUnitAt(i + j++) != $s) continue;
// Seen " is 0+ms" => We're done.
int ms = int.parse(s.substring(numberStartsAt, i + j - 2));
if (inMs) return ms;
return ms * 1000;
} else if (s.codeUnitAt(i + j) == $PERIOD) {
int dotAt = i + j;
j++;
if (!isNumber(s.codeUnitAt(i + j++))) continue;
while (isNumber(s.codeUnitAt(i + j))) {
j++;
}
if (s.codeUnitAt(i + j++) != $m) continue;
if (s.codeUnitAt(i + j++) != $s) continue;
// Seen " is 0+.0+ms" => We're done.
// int.parse(s.substring(numberStartsAt, i + j - 2));
int ms = int.parse(s.substring(numberStartsAt, dotAt));
if (inMs) return ms;
int fraction = int.parse(s.substring(dotAt + 1, i + j - 2));
while (fraction < 100) {
fraction *= 10;
}
while (fraction >= 1000) {
fraction ~/= 10;
}
return ms * 1000 + fraction;
} else {
continue;
}
}
usage("Didn't find any ms data in line '$s'.");
throw "usage should exit";
}
const int $SPACE = 32;
const int $PERIOD = 46;
const int $0 = 48;
const int $9 = 57;
const int $COLON = 58;
const int $_ = 95;
const int $i = 105;
const int $m = 109;
const int $n = 110;
const int $s = 115;
/// Check that format is like '0:00:00.000000: '.
bool isTimePrependedLine(String s) {
if (s.length < 15) return false;
int index = 0;
if (!isNumber(s.codeUnitAt(index++))) return false;
if (s.codeUnitAt(index++) != $COLON) return false;
if (!isNumber(s.codeUnitAt(index++))) return false;
if (!isNumber(s.codeUnitAt(index++))) return false;
if (s.codeUnitAt(index++) != $COLON) return false;
if (!isNumber(s.codeUnitAt(index++))) return false;
if (!isNumber(s.codeUnitAt(index++))) return false;
if (s.codeUnitAt(index++) != $PERIOD) return false;
for (int i = 0; i < 6; i++) {
if (!isNumber(s.codeUnitAt(index++))) return false;
}
if (s.codeUnitAt(index++) != $COLON) return false;
return true;
}
bool isNumber(int codeUnit) {
return codeUnit >= $0 && codeUnit <= $9;
}
String replaceNumbers(String s) {
StringBuffer sb = new StringBuffer();
bool lastWasNumber = false;
for (int i = 0; i < s.length; i++) {
int codeUnit = s.codeUnitAt(i);
if (isNumber(codeUnit)) {
if (!lastWasNumber) {
// Ignore number; replace with '_'.
sb.writeCharCode($_);
lastWasNumber = true;
}
} else {
sb.writeCharCode(codeUnit);
lastWasNumber = false;
}
}
return sb.toString();
}