[dart/fuzzer] DartFuzz Improvements
Included:
(1) Added more expression variants in generator.
(2) Added --isolates option in driver (enables parallel execution).
Change-Id: I4e04490acc3046fd4771dd07502ba0fcdceb7df5
Reviewed-on: https://dart-review.googlesource.com/c/77720
Reviewed-by: Alexander Thomas <athom@google.com>
Commit-Queue: Aart Bik <ajcbik@google.com>
diff --git a/runtime/tools/dartfuzz/README.md b/runtime/tools/dartfuzz/README.md
index dcf0b3f..cfd8b5e 100644
--- a/runtime/tools/dartfuzz/README.md
+++ b/runtime/tools/dartfuzz/README.md
@@ -32,6 +32,7 @@
dart dartfuzz_test.dart
run_dartfuzz_test.py [--help]
+ [--isolates ISOLATES ]
[--repeat REPEAT]
[--true_divergence]
[--mode1 MODE]
@@ -40,8 +41,9 @@
where
--help : prints help and exits
+ --isolates : number of isolates in the session (1 by default)
--repeat : number of tests to run (1000 by default)
- --show-stats : show session statistics (true by default)
+ --show-stats : show statistics during session (true by default)
--true-divergence : only report true divergences (true by default)
--dart-top : sets DART_TOP explicitly through command line
--mode1 : m1
diff --git a/runtime/tools/dartfuzz/dartfuzz.dart b/runtime/tools/dartfuzz/dartfuzz.dart
index 06790d0..9a36b2b 100644
--- a/runtime/tools/dartfuzz/dartfuzz.dart
+++ b/runtime/tools/dartfuzz/dartfuzz.dart
@@ -10,7 +10,7 @@
// Version of DartFuzz. Increase this each time changes are made
// to preserve the property that a given version of DartFuzz yields
// the same fuzzed program for a deterministic random seed.
-const String version = '1.0';
+const String version = '1.1';
// Restriction on statement and expression depths.
const int stmtDepth = 5;
@@ -58,7 +58,7 @@
// Interesting characters.
const interestingChars =
- 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#&()+-';
+ 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#&()+- ';
// Class that represents Dart types.
class DartType {
@@ -85,6 +85,13 @@
DartType.INT_STRING_MAP
];
+// Class that represents Dart library methods.
+class DartLib {
+ final String name;
+ final List<DartType> proto;
+ const DartLib(this.name, this.proto);
+}
+
// Naming conventions.
const varName = 'var';
const paramName = 'par';
@@ -375,16 +382,12 @@
emit('-${rand.nextInt(100)}');
}
- void emitInterestingInt() {
- int i = rand.nextInt(interestingIntegers.length);
- emit('${interestingIntegers[i]}');
- }
-
void emitInt() {
switch (rand.nextInt(4)) {
// favors small positive int
case 0:
- emitInterestingInt();
+ emit(
+ '${interestingIntegers[rand.nextInt(interestingIntegers.length)]}');
break;
case 1:
emitSmallNegativeInt();
@@ -420,71 +423,52 @@
}
void emitChar() {
- emit(interestingChars[rand.nextInt(interestingChars.length)]);
+ switch (rand.nextInt(10)) {
+ // favors regular char
+ case 0:
+ emit('\\u2665');
+ break;
+ case 1:
+ emit('\\u{1f600}'); // rune
+ break;
+ default:
+ emit(interestingChars[rand.nextInt(interestingChars.length)]);
+ break;
+ }
}
void emitString() {
- switch (rand.nextInt(4)) {
- // favors non-null
- case 0:
- emit('null');
- break;
- default:
- {
- emit("'");
- int l = rand.nextInt(8);
- for (int i = 0; i < l; i++) {
- emitChar();
- }
- emit("'");
- break;
- }
+ emit("'");
+ int l = rand.nextInt(8);
+ for (int i = 0; i < l; i++) {
+ emitChar();
}
+ emit("'");
}
void emitIntList() {
- switch (rand.nextInt(4)) {
- // favors non-null
- case 0:
- emit('null');
- break;
- default:
- {
- emit('[ ');
- int l = 1 + rand.nextInt(4);
- for (int i = 0; i < l; i++) {
- emitInt();
- if (i != (l - 1)) {
- emit(', ');
- }
- }
- emit(' ]');
- break;
- }
+ emit('[ ');
+ int l = 1 + rand.nextInt(4);
+ for (int i = 0; i < l; i++) {
+ emitInt();
+ if (i != (l - 1)) {
+ emit(', ');
+ }
}
+ emit(' ]');
}
void emitIntStringMap() {
- switch (rand.nextInt(4)) {
- // favors non-null
- case 0:
- emit('null');
- break;
- default:
- {
- emit('{ ');
- int l = 1 + rand.nextInt(4);
- for (int i = 0; i < l; i++) {
- emit('$i : ');
- emitString();
- if (i != (l - 1)) {
- emit(', ');
- }
- }
- emit(' }');
- break;
- }
+ emit('{ ');
+ int l = 1 + rand.nextInt(4);
+ for (int i = 0; i < l; i++) {
+ emit('$i : ');
+ emitString();
+ if (i != (l - 1)) {
+ emit(', ');
+ }
}
+ emit(' }');
}
void emitLiteral(DartType tp) {
@@ -548,28 +532,115 @@
}
void emitExprList(int depth, List<DartType> proto) {
+ emit('(');
for (int i = 1; i < proto.length; i++) {
emitExpr(depth, proto[i]);
if (i != (proto.length - 1)) {
emit(', ');
}
}
+ emit(')');
}
+ // Emit expression with unary operator: (~(x))
+ void emitUnaryExpr(int depth, DartType tp) {
+ if (tp == DartType.BOOL || tp == DartType.INT || tp == DartType.DOUBLE) {
+ emit('(');
+ emitUnaryOp(tp);
+ emit('(');
+ emitExpr(depth + 1, tp);
+ emit('))');
+ } else {
+ emitTerminal(tp); // resort to terminal
+ }
+ }
+
+ // Emit expression with binary operator: (x + y)
+ void emitBinaryExpr(int depth, DartType tp) {
+ if (tp == DartType.BOOL) {
+ // For boolean, allow type switch with relational op.
+ if (rand.nextInt(2) == 0) {
+ DartType deeper_tp = getType();
+ emit('(');
+ emitExpr(depth + 1, deeper_tp);
+ emitRelOp(deeper_tp);
+ emitExpr(depth + 1, deeper_tp);
+ emit(')');
+ return;
+ }
+ }
+ emit('(');
+ emitExpr(depth + 1, tp);
+ emitBinaryOp(tp);
+ emitExpr(depth + 1, tp);
+ emit(')');
+ }
+
+ // Emit expression with ternary operator: (b ? x : y)
+ void emitTernaryExpr(int depth, DartType tp) {
+ emit('(');
+ emitExpr(depth + 1, DartType.BOOL);
+ emit(' ? ');
+ emitExpr(depth + 1, tp);
+ emit(' : ');
+ emitExpr(depth + 1, tp);
+ emit(')');
+ }
+
+ // Emit expression with pre/post-increment/decrement operator: (x++)
+ void emitPreOrPostExpr(int depth, DartType tp) {
+ if (tp == DartType.INT) {
+ int r = rand.nextInt(2);
+ emit('(');
+ if (r == 0) emitPreOrPostOp(tp);
+ emitScalarVar(tp);
+ if (r == 1) emitPreOrPostOp(tp);
+ emit(')');
+ } else {
+ emitTerminal(tp); // resort to terminal
+ }
+ }
+
+ // Emit library call.
+ void emitLibraryCall(int depth, DartType tp) {
+ if (tp == DartType.INT_STRING_MAP) {
+ emitTerminal(tp); // resort to terminal
+ return;
+ }
+ DartLib lib = oneOf<DartLib>(getLibrary(tp));
+ List<DartType> proto = lib.proto;
+ // Receiver.
+ if (proto[0] != null) {
+ DartType deeper_tp = proto[0];
+ emit('(');
+ emitExpr(depth + 1, deeper_tp);
+ emit(').');
+ }
+ // Call.
+ emit('${lib.name}');
+ // Parameters.
+ if (proto.length == 1) {
+ emit('()');
+ } else if (proto[1] != null) {
+ emitExprList(depth + 1, proto);
+ }
+ }
+
+ // Helper for a method call.
bool pickedCall(
int depth, DartType tp, String name, List<List<DartType>> protos, int m) {
for (int i = m - 1; i >= 0; i--) {
if (tp == protos[i][0]) {
- emit('$name$i(');
+ emit('$name$i');
emitExprList(depth + 1, protos[i]);
- emit(')');
return true;
}
}
return false;
}
- void emitCall(int depth, DartType tp) {
+ // Emit method call within the program.
+ void emitMethodCall(int depth, DartType tp) {
// Only call backward to avoid infinite recursion.
if (currentClass == null) {
// Outside a class: call backward in global methods.
@@ -591,70 +662,7 @@
emitTerminal(tp); // resort to terminal.
}
- // Emit expression with unary operator: (~(x))
- void emitUnaryExpr(int depth, DartType tp) {
- if (tp == DartType.BOOL || tp == DartType.INT || tp == DartType.DOUBLE) {
- emit('(');
- emitUnaryOp(tp);
- emit('(');
- emitExpr(depth + 1, tp);
- emit('))');
- } else {
- emitTerminal(tp); // resort to terminal
- }
- }
-
- // Emit expression with binary operator: (x + y)
- void emitBinaryExpr(int depth, DartType tp) {
- if (tp == DartType.BOOL || tp == DartType.INT || tp == DartType.DOUBLE) {
- emit('(');
- emitExpr(depth + 1, tp);
- emitBinaryOp(tp);
- emitExpr(depth + 1, tp);
- emit(')');
- } else {
- emitTerminal(tp); // resort to terminal
- }
- }
-
- // Emit expression with pre/post-increment/decrement operator: (x++)
- void emitPreOrPostExpr(int depth, DartType tp) {
- if (tp == DartType.INT) {
- int r = rand.nextInt(2);
- emit('(');
- if (r == 0) emitPreOrPostOp(tp);
- emitScalarVar(tp);
- if (r == 1) emitPreOrPostOp(tp);
- emit(')');
- } else {
- emitTerminal(tp); // resort to terminal
- }
- }
-
- // Emit type converting expression.
- void emitTypeConv(int depth, DartType tp) {
- if (tp == DartType.BOOL) {
- DartType deeper_tp = getType();
- emit('(');
- emitExpr(depth + 1, deeper_tp);
- emitRelOp(deeper_tp);
- emitExpr(depth + 1, deeper_tp);
- emit(')');
- } else if (tp == DartType.INT) {
- emit('(');
- emitExpr(depth + 1, DartType.DOUBLE);
- emit(').toInt()');
- } else if (tp == DartType.DOUBLE) {
- emit('(');
- emitExpr(depth + 1, DartType.INT);
- emit(').toDouble()');
- } else {
- emitTerminal(tp); // resort to terminal
- }
- }
-
// Emit expression.
- // TODO: add many more constructs
void emitExpr(int depth, DartType tp) {
// Continuing nested expressions becomes less likely as the depth grows.
if (rand.nextInt(depth + 1) > exprDepth) {
@@ -662,7 +670,7 @@
return;
}
// Possibly nested expression.
- switch (rand.nextInt(6)) {
+ switch (rand.nextInt(7)) {
case 0:
emitUnaryExpr(depth, tp);
break;
@@ -670,13 +678,16 @@
emitBinaryExpr(depth, tp);
break;
case 2:
- emitPreOrPostExpr(depth, tp);
+ emitTernaryExpr(depth, tp);
break;
case 3:
- emitTypeConv(depth, tp);
+ emitPreOrPostExpr(depth, tp);
break;
case 4:
- emitCall(depth, tp);
+ emitLibraryCall(depth, tp);
+ break;
+ case 5:
+ emitMethodCall(depth, tp);
break;
default:
emitTerminal(tp);
@@ -688,11 +699,6 @@
// Operators.
//
- // Emit one of the given choices.
- T oneOf<T>(List<T> choices) {
- return choices[rand.nextInt(choices.length)];
- }
-
// Emit same type in-out assignment operator.
void emitAssignOp(DartType tp) {
if (tp == DartType.INT) {
@@ -707,12 +713,14 @@
' ^= ',
' >>= ',
' <<= ',
+ ' ??= ',
' = '
]));
} else if (tp == DartType.DOUBLE) {
- emit(oneOf(const <String>[' += ', ' -= ', ' *= ', ' /= ', ' = ']));
+ emit(oneOf(
+ const <String>[' += ', ' -= ', ' *= ', ' /= ', ' ??= ', ' = ']));
} else {
- emit(' = ');
+ emit(oneOf(const <String>[' ??= ', ' = ']));
}
}
@@ -744,16 +752,19 @@
' | ',
' ^ ',
' >> ',
- ' << '
+ ' << ',
+ ' ?? '
]));
} else if (tp == DartType.DOUBLE) {
- emit(oneOf(const <String>[' + ', ' - ', ' * ', ' / ']));
+ emit(oneOf(const <String>[' + ', ' - ', ' * ', ' / ', ' ?? ']));
+ } else if (tp == DartType.STRING || tp == DartType.INT_LIST) {
+ emit(oneOf(const <String>[' + ', ' ?? ']));
} else {
- assert(false);
+ emit(' ?? ');
}
}
- // Emit increment operator.
+ // Emit same type in-out increment operator.
void emitPreOrPostOp(DartType tp) {
if (tp == DartType.INT) {
emit(oneOf(const <String>['++', '--']));
@@ -772,9 +783,105 @@
}
//
+ // Library methods.
+ //
+
+ // Get list of library methods, organized by return type.
+ // Proto list:
+ // [ receiver-type (null denotes none),
+ // param1 type (null denotes getter),
+ // param2 type,
+ // ...
+ // ]
+ List<DartLib> getLibrary(DartType tp) {
+ if (tp == DartType.BOOL) {
+ return const [
+ DartLib('isEven', [DartType.INT, null]),
+ DartLib('isOdd', [DartType.INT, null]),
+ DartLib('isEmpty', [DartType.STRING, null]),
+ DartLib('isEmpty', [DartType.INT_STRING_MAP, null]),
+ DartLib('isNotEmpty', [DartType.STRING, null]),
+ DartLib('isNotEmpty', [DartType.INT_STRING_MAP, null]),
+ DartLib('endsWith', [DartType.STRING, DartType.STRING]),
+ DartLib('remove', [DartType.INT_LIST, DartType.INT]),
+ DartLib('containsValue', [DartType.INT_STRING_MAP, DartType.STRING]),
+ DartLib('containsKey', [DartType.INT_STRING_MAP, DartType.INT]),
+ ];
+ } else if (tp == DartType.INT) {
+ return const [
+ DartLib('bitLength', [DartType.INT, null]),
+ DartLib('sign', [DartType.INT, null]),
+ DartLib('abs', [DartType.INT]),
+ DartLib('round', [DartType.INT]),
+ DartLib('round', [DartType.DOUBLE]),
+ DartLib('floor', [DartType.INT]),
+ DartLib('floor', [DartType.DOUBLE]),
+ DartLib('ceil', [DartType.INT]),
+ DartLib('ceil', [DartType.DOUBLE]),
+ DartLib('truncate', [DartType.INT]),
+ DartLib('truncate', [DartType.DOUBLE]),
+ DartLib('toInt', [DartType.DOUBLE]),
+ DartLib('toUnsigned', [DartType.INT, DartType.INT]),
+ DartLib('toSigned', [DartType.INT, DartType.INT]),
+ DartLib('modInverse', [DartType.INT, DartType.INT]),
+ DartLib('modPow', [DartType.INT, DartType.INT, DartType.INT]),
+ DartLib('length', [DartType.STRING, null]),
+ DartLib('length', [DartType.INT_LIST, null]),
+ DartLib('length', [DartType.INT_STRING_MAP, null]),
+ DartLib('codeUnitAt', [DartType.STRING, DartType.INT]),
+ DartLib('compareTo', [DartType.STRING, DartType.STRING]),
+ DartLib('removeLast', [DartType.INT_LIST]),
+ DartLib('removeAt', [DartType.INT_LIST, DartType.INT]),
+ DartLib('indexOf', [DartType.INT_LIST, DartType.INT]),
+ DartLib('lastIndexOf', [DartType.INT_LIST, DartType.INT]),
+ ];
+ } else if (tp == DartType.DOUBLE) {
+ return const [
+ DartLib('sign', [DartType.DOUBLE, null]),
+ DartLib('abs', [DartType.DOUBLE]),
+ DartLib('toDouble', [DartType.INT]),
+ DartLib('roundToDouble', [DartType.INT]),
+ DartLib('roundToDouble', [DartType.DOUBLE]),
+ DartLib('floorToDouble', [DartType.INT]),
+ DartLib('floorToDouble', [DartType.DOUBLE]),
+ DartLib('ceilToDouble', [DartType.INT]),
+ DartLib('ceilToDouble', [DartType.DOUBLE]),
+ DartLib('truncateToDouble', [DartType.INT]),
+ DartLib('truncateToDouble', [DartType.DOUBLE]),
+ DartLib('remainder', [DartType.DOUBLE, DartType.DOUBLE]),
+ ];
+ } else if (tp == DartType.STRING) {
+ return const [
+ DartLib('toString', [DartType.BOOL]),
+ DartLib('toString', [DartType.INT]),
+ DartLib('toString', [DartType.DOUBLE]),
+ DartLib('toRadixString', [DartType.INT, DartType.INT]),
+ DartLib('trim', [DartType.STRING]),
+ DartLib('trimLeft', [DartType.STRING]),
+ DartLib('trimRight', [DartType.STRING]),
+ DartLib('toLowerCase', [DartType.STRING]),
+ DartLib('toUpperCase', [DartType.STRING]),
+ DartLib('substring', [DartType.STRING, DartType.INT]),
+ DartLib('padLeft', [DartType.STRING, DartType.INT]),
+ DartLib('padRight', [DartType.STRING, DartType.INT]),
+ DartLib('replaceRange',
+ [DartType.STRING, DartType.INT, DartType.INT, DartType.STRING]),
+ DartLib('remove', [DartType.INT_STRING_MAP, DartType.INT]),
+ ];
+ } else if (tp == DartType.INT_LIST) {
+ return const [
+ DartLib('sublist', [DartType.INT_LIST, DartType.INT]),
+ ];
+ } else {
+ assert(false);
+ }
+ }
+
+ //
// Types.
//
+ // Get a random value type.
DartType getType() {
switch (rand.nextInt(6)) {
case 0:
@@ -854,6 +961,11 @@
}
}
+ // Emits one of the given choices.
+ T oneOf<T>(List<T> choices) {
+ return choices[rand.nextInt(choices.length)];
+ }
+
// Random seed used to generate program.
final int seed;
@@ -872,7 +984,7 @@
// Types of global variables.
List<DartType> globalVars;
- // Prototypes of all global functions (first element is return type).
+ // Prototypes of all global methods (first element is return type).
List<List<DartType>> globalMethods;
// Types of fields over all classes.
@@ -908,7 +1020,7 @@
new DartFuzz(seed, file).run();
file.closeSync();
} catch (e) {
- print('Usage: dart dartfuzz.dart [OPTIONS] FILENAME');
- print(parser.usage);
+ print('Usage: dart dartfuzz.dart [OPTIONS] FILENAME\n${parser.usage}\n$e');
+ exitCode = 255;
}
}
diff --git a/runtime/tools/dartfuzz/dartfuzz_test.dart b/runtime/tools/dartfuzz/dartfuzz_test.dart
index bfde9ae..87af806 100644
--- a/runtime/tools/dartfuzz/dartfuzz_test.dart
+++ b/runtime/tools/dartfuzz/dartfuzz_test.dart
@@ -4,6 +4,7 @@
import 'dart:convert';
import 'dart:io';
+import 'dart:isolate';
import 'dart:math';
import 'package:args/args.dart';
@@ -14,31 +15,14 @@
const sigkill = 9;
const timeout = 30; // in seconds
-// Supported modes.
-const List<String> modes = [
- 'jit-debug-ia32',
- 'jit-debug-x64',
- 'jit-debug-arm32',
- 'jit-debug-arm64',
- 'aot-debug-x64',
- 'aot-debug-arm64',
- 'jit-ia32',
- 'jit-x64',
- 'jit-arm32',
- 'jit-arm64',
- 'aot-x64',
- 'aot-arm64',
- 'js'
-];
-
// Exit code of running a test.
enum ResultCode { success, timeout, error }
/// Result of running a test.
class TestResult {
TestResult(this.code, this.output);
- ResultCode code;
- String output;
+ final ResultCode code;
+ final String output;
}
/// Command runner.
@@ -61,15 +45,17 @@
/// Abstraction for running one test in a particular mode.
abstract class TestRunner {
- String description();
- TestResult run(String fileName);
+ TestResult run();
+ String description;
// Factory.
static TestRunner getTestRunner(
- Map<String, String> env, String top, String mode) {
- if (mode.startsWith('jit')) return new TestRunnerJIT(env, top, mode);
- if (mode.startsWith('aot')) return new TestRunnerAOT(env, top, mode);
- if (mode.startsWith('js')) return new TestRunnerJS(env, top, mode);
+ String mode, String top, String tmp, Map<String, String> env) {
+ if (mode.startsWith('jit'))
+ return new TestRunnerJIT(getTag(mode), top, tmp, env);
+ if (mode.startsWith('aot'))
+ return new TestRunnerAOT(getTag(mode), top, tmp, env);
+ if (mode.startsWith('js')) return new TestRunnerJS(top, tmp, env);
throw ('unknown runner in mode: $mode');
}
@@ -89,116 +75,115 @@
/// Concrete test runner of Dart JIT.
class TestRunnerJIT implements TestRunner {
- TestRunnerJIT(Map<String, String> e, String top, String mode) {
- tag = TestRunner.getTag(mode);
+ TestRunnerJIT(String tag, String top, String tmp, Map<String, String> e) {
+ description = 'JIT-${tag}';
dart = '$top/out/$tag/dart';
+ fileName = '$tmp/fuzz.dart';
env = e;
}
- String description() {
- return "JIT-${tag}";
+
+ TestResult run() {
+ return runCommand([dart, fileName], env);
}
- TestResult run(String fileName) {
- return runCommand(['$dart', fileName], env);
- }
-
- String tag;
+ String description;
String dart;
+ String fileName;
Map<String, String> env;
}
/// Concrete test runner of Dart AOT.
class TestRunnerAOT implements TestRunner {
- TestRunnerAOT(Map<String, String> e, String top, String mode) {
- tag = TestRunner.getTag(mode);
+ TestRunnerAOT(String tag, String top, String tmp, Map<String, String> e) {
+ description = 'AOT-${tag}';
precompiler = '$top/pkg/vm/tool/precompiler2';
dart = '$top/pkg/vm/tool/dart_precompiled_runtime2';
+ fileName = '$tmp/fuzz.dart';
+ snapshot = '$tmp/snapshot';
env = Map<String, String>.from(e);
env['DART_CONFIGURATION'] = tag;
}
- String description() {
- return "AOT-${tag}";
- }
- TestResult run(String fileName) {
- TestResult result = runCommand(['$precompiler', fileName, 'snapshot'], env);
+ TestResult run() {
+ TestResult result = runCommand([precompiler, fileName, snapshot], env);
if (result.code != ResultCode.success) {
return result;
}
- return runCommand(['$dart', 'snapshot'], env);
+ return runCommand([dart, snapshot], env);
}
- String tag;
+ String description;
String precompiler;
String dart;
+ String fileName;
+ String snapshot;
Map<String, String> env;
}
/// Concrete test runner of Dart2JS.
class TestRunnerJS implements TestRunner {
- TestRunnerJS(Map<String, String> e, String top, String mode) {
+ TestRunnerJS(String top, String tmp, Map<String, String> e) {
+ description = 'Dart2JS';
dart2js = '$top/out/ReleaseX64/dart-sdk/bin/dart2js';
+ fileName = '$tmp/fuzz.dart';
+ js = '$tmp/out.js';
env = e;
}
- String description() {
- return "Dart2JS";
- }
- TestResult run(String fileName) {
- TestResult result = runCommand(['$dart2js', fileName], env);
+ TestResult run() {
+ TestResult result = runCommand([dart2js, fileName, '-o', js], env);
if (result.code != ResultCode.success) {
return result;
}
- return runCommand(['nodejs', 'out.js'], env);
+ return runCommand(['nodejs', js], env);
}
+ String description;
String dart2js;
+ String fileName;
+ String js;
Map<String, String> env;
}
-/// Class to run a fuzz testing session.
+/// Class to run fuzz testing.
class DartFuzzTest {
DartFuzzTest(this.env, this.repeat, this.trueDivergence, this.showStats,
this.top, this.mode1, this.mode2);
- bool runSession() {
- setupSession();
+ bool run() {
+ setup();
- print('\n**\n**** Dart Fuzz Testing\n**\n');
- print('Fuzz Version : ${version}');
- print('#Tests : ${repeat}');
- print('Exec-Mode 1 : ${runner1.description()}');
- print('Exec-Mode 2 : ${runner2.description()}');
- print('Dart Dev : ${top}');
- print('Orig Dir : ${orgDir.path}');
- print('Temp Dir : ${tmpDir.path}\n');
+ print('\nRun isolate: ${runner1.description} vs. '
+ '${runner2.description} in ${tmpDir.path}');
- showStatistics();
+ if (showStats) {
+ showStatistics();
+ }
for (int i = 0; i < repeat; i++) {
numTests++;
seed = rand.nextInt(1 << 32);
generateTest();
runTest();
- showStatistics();
+ if (showStats) {
+ showStatistics();
+ }
}
- cleanupSession();
- if (numDivergences != 0) {
- print('\n\nfailure\n');
- return false;
- }
- print('\n\nsuccess\n');
- return true;
+ print('\nDone isolate: ${runner1.description} vs. '
+ '${runner2.description} in ${tmpDir.path}');
+ showStatistics();
+ print('');
+
+ cleanup();
+ return numDivergences == 0;
}
- void setupSession() {
+ void setup() {
rand = new Random();
- orgDir = Directory.current;
tmpDir = Directory.systemTemp.createTempSync('dart_fuzz');
- Directory.current = tmpDir;
- fileName = 'fuzz.dart';
- runner1 = TestRunner.getTestRunner(env, top, mode1);
- runner2 = TestRunner.getTestRunner(env, top, mode2);
+ fileName = '${tmpDir.path}/fuzz.dart';
+ runner1 = TestRunner.getTestRunner(mode1, top, tmpDir.path, env);
+ runner2 = TestRunner.getTestRunner(mode2, top, tmpDir.path, env);
numTests = 0;
numSuccess = 0;
numNotRun = 0;
@@ -206,16 +191,13 @@
numDivergences = 0;
}
- void cleanupSession() {
- Directory.current = orgDir;
+ void cleanup() {
tmpDir.delete(recursive: true);
}
void showStatistics() {
- if (showStats) {
- stdout.write('\rTests: $numTests Success: $numSuccess Not-Run: '
- '$numNotRun: Time-Out: $numTimeOut Divergences: $numDivergences');
- }
+ stdout.write('\rTests: $numTests Success: $numSuccess Not-Run: '
+ '$numNotRun: Time-Out: $numTimeOut Divergences: $numDivergences');
}
void generateTest() {
@@ -225,8 +207,8 @@
}
void runTest() {
- TestResult result1 = runner1.run(fileName);
- TestResult result2 = runner2.run(fileName);
+ TestResult result1 = runner1.run();
+ TestResult result2 = runner2.run();
checkDivergence(result1, result2);
}
@@ -269,7 +251,8 @@
void reportDivergence(
TestResult result1, TestResult result2, bool outputDivergence) {
numDivergences++;
- print('\n\nDIVERGENCE on generated program $version:$seed\n');
+ print('\n\nDIVERGENCE $version:$seed : ${runner1.description} vs. '
+ '${runner2.description}\n');
if (outputDivergence) {
print('out1:\n${result1.output}\n\nout2:\n${result2.output}\n');
}
@@ -284,9 +267,8 @@
final String mode1;
final String mode2;
- // Session.
+ // Test.
Random rand;
- Directory orgDir;
Directory tmpDir;
String fileName;
TestRunner runner1;
@@ -301,38 +283,119 @@
int numDivergences;
}
-// Picks a mode (command line or random).
-String getMode(String mode, String other) {
- // Random when not set.
- if (mode == null || mode == '') {
- // Pick a mode at random (not JS), different from other.
- Random rand = new Random();
- do {
- mode = modes[rand.nextInt(modes.length - 1)];
- } while (mode == other);
- }
- // Verify mode.
- if (modes.contains(mode)) {
- return mode;
- }
- throw ('unknown mode: $mode');
-}
+/// Class to start fuzz testing session.
+class DartFuzzTestSession {
+ DartFuzzTestSession(this.isolates, this.repeat, this.trueDivergence,
+ this.showStats, String tp, this.mode1, this.mode2)
+ : top = getTop(tp) {}
-// Picks a top directory (command line, environment, or current).
-String getTop(String top) {
- if (top == null || top == '') {
- top = Platform.environment['DART_TOP'];
+ start() async {
+ print('\n**\n**** Dart Fuzz Testing Session\n**\n');
+ print('Fuzz Version : ${version}');
+ print('Isolates : ${isolates}');
+ print('Tests : ${repeat}');
+ print('True Divergence : ${trueDivergence}');
+ print('Show Stats : ${showStats}');
+ print('Dart Dev : ${top}');
+ // Fork.
+ List<ReceivePort> ports = new List();
+ for (int i = 0; i < isolates; i++) {
+ ReceivePort r = new ReceivePort();
+ ports.add(r);
+ port = r.sendPort;
+ await Isolate.spawn(run, this);
+ }
+ // Join.
+ bool success = true;
+ for (int i = 0; i < isolates; i++) {
+ var x = await ports[i].first;
+ success = success && x;
+ }
+ if (success) {
+ print('\nsuccess\n');
+ } else {
+ print('\nfailure\n');
+ exitCode = 1;
+ }
}
- if (top == null || top == '') {
- top = Directory.current.path;
+
+ static run(DartFuzzTestSession session) {
+ bool success = false;
+ try {
+ final m1 = getMode(session.mode1, null);
+ final m2 = getMode(session.mode2, m1);
+ final fuzz = new DartFuzzTest(Platform.environment, session.repeat,
+ session.trueDivergence, session.showStats, session.top, m1, m2);
+ success = fuzz.run();
+ } catch (e) {
+ print('Isolate: $e');
+ }
+ session.port.send(success);
}
- return top;
+
+ // Picks a top directory (command line, environment, or current).
+ static String getTop(String top) {
+ if (top == null || top == '') {
+ top = Platform.environment['DART_TOP'];
+ }
+ if (top == null || top == '') {
+ top = Directory.current.path;
+ }
+ return top;
+ }
+
+ // Picks a mode (command line or random).
+ static String getMode(String mode, String other) {
+ // Random when not set.
+ if (mode == null || mode == '') {
+ // Pick a mode at random (not JS), different from other.
+ Random rand = new Random();
+ do {
+ mode = modes[rand.nextInt(modes.length - 1)];
+ } while (mode == other);
+ }
+ // Verify mode.
+ if (modes.contains(mode)) {
+ return mode;
+ }
+ throw ('unknown mode: $mode');
+ }
+
+ // Context.
+ final int isolates;
+ final int repeat;
+ final bool trueDivergence;
+ final bool showStats;
+ final String top;
+ final String mode1;
+ final String mode2;
+
+ // Passes each port to isolate.
+ SendPort port;
+
+ // Supported modes.
+ static const List<String> modes = [
+ 'jit-debug-ia32',
+ 'jit-debug-x64',
+ 'jit-debug-arm32',
+ 'jit-debug-arm64',
+ 'aot-debug-x64',
+ 'aot-debug-arm64',
+ 'jit-ia32',
+ 'jit-x64',
+ 'jit-arm32',
+ 'jit-arm64',
+ 'aot-x64',
+ 'aot-arm64',
+ 'js'
+ ];
}
/// Main driver for a fuzz testing session.
main(List<String> arguments) {
// Set up argument parser.
final parser = new ArgParser()
+ ..addOption('isolates', help: 'number of isolates to use', defaultsTo: '1')
..addOption('repeat', help: 'number of tests to run', defaultsTo: '1000')
..addFlag('true-divergence',
negatable: true, help: 'only report true divergences', defaultsTo: true)
@@ -342,20 +405,18 @@
..addOption('mode1', help: 'execution mode 1')
..addOption('mode2', help: 'execution mode 2');
- // Start fuzz testing session.
+ // Starts fuzz testing session.
try {
final results = parser.parse(arguments);
- final repeat = int.parse(results['repeat']);
- final trueDivergence = results['true-divergence'];
- final showStats = results['show-stats'];
- final mode1 = getMode(results['mode1'], null);
- final mode2 = getMode(results['mode2'], mode1);
- final top = getTop(results['dart-top']);
- final session = new DartFuzzTest(Platform.environment, repeat,
- trueDivergence, showStats, top, mode1, mode2);
- if (!session.runSession()) {
- exitCode = 1;
- }
+ new DartFuzzTestSession(
+ int.parse(results['isolates']),
+ int.parse(results['repeat']),
+ results['true-divergence'],
+ results['show-stats'],
+ results['dart-top'],
+ results['mode1'],
+ results['mode2'])
+ .start();
} catch (e) {
print('Usage: dart dartfuzz_test.dart [OPTIONS]\n${parser.usage}\n$e');
exitCode = 255;
