[dart/fuzzer] Introduce time-based test runs
Rationale:
To maximize CPU utilitization during fuzzing runs,
we now use a time-out on all the individual fuzz
testing runs (rather than repeat based). This way,
the "faster" tests will cover more ground, while
the "slower" straggler tests will not break the test
with unnecessary time-outs at cluster level.
Change-Id: I5f0f9646da3fbd8b361a7ea43444b1896a45b57f
Reviewed-on: https://dart-review.googlesource.com/c/84047
Commit-Queue: Aart Bik <ajcbik@google.com>
Reviewed-by: Alexander Thomas <athom@google.com>
diff --git a/runtime/tools/dartfuzz/README.md b/runtime/tools/dartfuzz/README.md
index ca5d676..0cb6537 100644
--- a/runtime/tools/dartfuzz/README.md
+++ b/runtime/tools/dartfuzz/README.md
@@ -34,6 +34,7 @@
run_dartfuzz_test.py [--help]
[--isolates ISOLATES ]
[--repeat REPEAT]
+ [--time TIME]
[--true_divergence]
[--mode1 MODE]
[--mode2 MODE]
@@ -43,6 +44,7 @@
--help : prints help and exits
--isolates : number of isolates in the session (1 by default)
--repeat : number of tests to run (1000 by default)
+ --time : time limit in seconds (none by default)
--show-stats : show statistics during session (true by default)
--true-divergence : only report true divergences (true by default)
--dart-top : sets DART_TOP explicitly through command line
diff --git a/runtime/tools/dartfuzz/dartfuzz_test.dart b/runtime/tools/dartfuzz/dartfuzz_test.dart
index 03d7146..a36b858 100644
--- a/runtime/tools/dartfuzz/dartfuzz_test.dart
+++ b/runtime/tools/dartfuzz/dartfuzz_test.dart
@@ -13,7 +13,7 @@
const debug = false;
const sigkill = 9;
-const timeout = 30; // in seconds
+const timeout = 60; // in seconds
// Exit code of running a test.
enum ResultCode { success, timeout, error }
@@ -198,8 +198,8 @@
/// Class to run fuzz testing.
class DartFuzzTest {
- DartFuzzTest(this.env, this.repeat, this.trueDivergence, this.showStats,
- this.top, this.mode1, this.mode2);
+ DartFuzzTest(this.env, this.repeat, this.time, this.trueDivergence,
+ this.showStats, this.top, this.mode1, this.mode2);
int run() {
setup();
@@ -216,8 +216,10 @@
runTest();
if (showStats) {
showStatistics();
- } else if ((i & 31) == 31) {
- print('\n${isolate}: busy @${numTests}....');
+ }
+ // Timeout?
+ if (timeIsUp()) {
+ break;
}
}
@@ -238,6 +240,11 @@
isolate = 'Isolate (${tmpDir.path}) '
'${runner1.description} - ${runner2.description}';
+ start_time = new DateTime.now().millisecondsSinceEpoch;
+ current_time = start_time;
+ report_time = start_time;
+ end_time = start_time + max(0, time - timeout) * 1000;
+
numTests = 0;
numSuccess = 0;
numNotRun = 0;
@@ -245,6 +252,22 @@
numDivergences = 0;
}
+ bool timeIsUp() {
+ if (time > 0) {
+ current_time = new DateTime.now().millisecondsSinceEpoch;
+ if (current_time > end_time) {
+ return true;
+ }
+ // Report every 10 minutes.
+ if ((current_time - report_time) > (10 * 60 * 1000)) {
+ print(
+ '\n${isolate}: busy @${numTests} ${current_time - start_time} seconds....');
+ report_time = current_time;
+ }
+ }
+ return false;
+ }
+
void cleanup() {
tmpDir.delete(recursive: true);
}
@@ -315,6 +338,7 @@
// Context.
final Map<String, String> env;
final int repeat;
+ final int time;
final bool trueDivergence;
final bool showStats;
final String top;
@@ -330,6 +354,12 @@
String isolate;
int seed;
+ // Timing
+ int start_time;
+ int current_time;
+ int report_time;
+ int end_time;
+
// Stats.
int numTests;
int numSuccess;
@@ -340,8 +370,8 @@
/// Class to start fuzz testing session.
class DartFuzzTestSession {
- DartFuzzTestSession(this.isolates, this.repeat, this.trueDivergence,
- this.showStats, String tp, this.mode1, this.mode2)
+ DartFuzzTestSession(this.isolates, this.repeat, this.time,
+ this.trueDivergence, this.showStats, String tp, this.mode1, this.mode2)
: top = getTop(tp) {}
start() async {
@@ -349,6 +379,11 @@
print('Fuzz Version : ${version}');
print('Isolates : ${isolates}');
print('Tests : ${repeat}');
+ if (time > 0) {
+ print('Time : ${time} seconds');
+ } else {
+ print('Time : unlimited');
+ }
print('True Divergence : ${trueDivergence}');
print('Show Stats : ${showStats}');
print('Dart Dev : ${top}');
@@ -379,8 +414,15 @@
try {
final m1 = getMode(session.mode1, null);
final m2 = getMode(session.mode2, m1);
- final fuzz = new DartFuzzTest(Platform.environment, session.repeat,
- session.trueDivergence, session.showStats, session.top, m1, m2);
+ final fuzz = new DartFuzzTest(
+ Platform.environment,
+ session.repeat,
+ session.time,
+ session.trueDivergence,
+ session.showStats,
+ session.top,
+ m1,
+ m2);
divergences = fuzz.run();
} catch (e) {
print('Isolate: $e');
@@ -420,6 +462,7 @@
// Context.
final int isolates;
final int repeat;
+ final int time;
final bool trueDivergence;
final bool showStats;
final String top;
@@ -466,6 +509,7 @@
final parser = new ArgParser()
..addOption('isolates', help: 'number of isolates to use', defaultsTo: '1')
..addOption('repeat', help: 'number of tests to run', defaultsTo: '1000')
+ ..addOption('time', help: 'time limit in seconds', defaultsTo: '0')
..addFlag('true-divergence',
negatable: true, help: 'only report true divergences', defaultsTo: true)
..addFlag('show-stats',
@@ -491,6 +535,7 @@
new DartFuzzTestSession(
int.parse(results['isolates']),
int.parse(results['repeat']),
+ int.parse(results['time']),
results['true-divergence'],
results['show-stats'],
results['dart-top'],
diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json
index 8e8879b..8732eea 100644
--- a/tools/bots/test_matrix.json
+++ b/tools/bots/test_matrix.json
@@ -1766,8 +1766,8 @@
"--isolates",
"8",
"--no-show-stats",
- "--repeat",
- "80"
+ "--time",
+ "2700"
],
"shards": 10,
"fileset": "fuzzer"
