Update tools and pkg/smith directories on analyzer-0.33 branch to match origin/master.
This should hopefully be sufficient to get bots passing on the
analyzer-0.33 branch.
Updated to origin/master as of c987c9e5a7e6f0c1158749e8f6cf694605ab48fb.
Change-Id: Icb24d30dcbb0aa525c0598973a1090385a2f3d73
Reviewed-on: https://dart-review.googlesource.com/c/86682
Reviewed-by: Paul Berry <paulberry@google.com>
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>
diff --git a/pkg/smith/lib/configuration.dart b/pkg/smith/lib/configuration.dart
index 9bd0e37..ea0b8d1 100644
--- a/pkg/smith/lib/configuration.dart
+++ b/pkg/smith/lib/configuration.dart
@@ -259,7 +259,6 @@
useAnalyzerCfe: boolOption("use-cfe"),
useAnalyzerFastaParser: boolOption("analyzer-use-fasta-parser"),
useBlobs: boolOption("use-blobs"),
- keepGeneratedFiles: boolOption("keep-generated-files"),
useDart2JSWithKernel: boolOption("dart2js-with-kernel"),
useDart2JSOldFrontEnd: boolOption("dart2js-old-frontend"),
useFastStartup: boolOption("fast-startup"),
@@ -315,9 +314,6 @@
// TODO(rnystrom): What is this?
final bool useBlobs;
- // Keep generated files (instead of deleting them).
- final bool keepGeneratedFiles;
-
// TODO(rnystrom): Remove these when Dart 1.0 is no longer supported.
final bool useDart2JSWithKernel;
final bool useDart2JSOldFrontEnd;
@@ -343,7 +339,6 @@
bool useAnalyzerCfe,
bool useAnalyzerFastaParser,
bool useBlobs,
- bool keepGeneratedFiles,
bool useDart2JSWithKernel,
bool useDart2JSOldFrontEnd,
bool useFastStartup,
@@ -362,7 +357,6 @@
useAnalyzerCfe = useAnalyzerCfe ?? false,
useAnalyzerFastaParser = useAnalyzerFastaParser ?? false,
useBlobs = useBlobs ?? false,
- keepGeneratedFiles = keepGeneratedFiles ?? false,
useDart2JSWithKernel = useDart2JSWithKernel ?? false,
useDart2JSOldFrontEnd = useDart2JSOldFrontEnd ?? false,
useFastStartup = useFastStartup ?? false,
@@ -390,7 +384,6 @@
useAnalyzerCfe == other.useAnalyzerCfe &&
useAnalyzerFastaParser == other.useAnalyzerFastaParser &&
useBlobs == other.useBlobs &&
- keepGeneratedFiles == other.keepGeneratedFiles &&
useDart2JSWithKernel == other.useDart2JSWithKernel &&
useDart2JSOldFrontEnd == other.useDart2JSOldFrontEnd &&
useFastStartup == other.useFastStartup &&
@@ -425,8 +418,7 @@
(useFastStartup ? 2048 : 0) ^
(useHotReload ? 4096 : 0) ^
(useHotReloadRollback ? 8192 : 0) ^
- (useSdk ? 16384 : 0) ^
- (keepGeneratedFiles ? 32768 : 0);
+ (useSdk ? 16384 : 0);
String toString() {
var buffer = new StringBuffer();
@@ -452,7 +444,6 @@
if (useAnalyzerCfe) fields.add("use-cfe");
if (useAnalyzerFastaParser) fields.add("analyzer-use-fasta-parser");
if (useBlobs) fields.add("use-blobs");
- if (keepGeneratedFiles) fields.add("keep-generated-files");
if (useDart2JSWithKernel) fields.add("dart2js-with-kernel");
if (useDart2JSOldFrontEnd) fields.add("dart2js-old-frontend");
if (useFastStartup) fields.add("fast-startup");
@@ -516,10 +507,6 @@
if (useBlobs || other.useBlobs) {
fields.add("useBlobs $useBlobs ${other.useBlobs}");
}
- if (keepGeneratedFiles || other.keepGeneratedFiles) {
- fields.add(
- "keepGeneratedFiles $keepGeneratedFiles ${other.keepGeneratedFiles}");
- }
if (useDart2JSWithKernel || other.useDart2JSWithKernel) {
fields.add("useDart2JSWithKernel "
"$useDart2JSWithKernel ${other.useDart2JSWithKernel}");
diff --git a/tools/FAKE_COMMITS b/tools/FAKE_COMMITS
index fbe07a6..14748a9 100644
--- a/tools/FAKE_COMMITS
+++ b/tools/FAKE_COMMITS
@@ -24,3 +24,6 @@
Trigger mirroring of github repository
Force build after DEPS format revert
Force build while trybots are broken, to check builders for brokenness.
+
+Analyzer branch commits:
+Force build on new analyzer-branch linux build with new workflow
\ No newline at end of file
diff --git a/tools/android/VERSION_LINUX_NDK b/tools/android/VERSION_LINUX_NDK
index 9fff8ff..4fac77b 100644
--- a/tools/android/VERSION_LINUX_NDK
+++ b/tools/android/VERSION_LINUX_NDK
@@ -1 +1 @@
-50bb526ee77fee88b9382d2bf48e7399751b98ae
+e626c47cb82a7439b0eda03ac6e0e9e1e41c6093
diff --git a/tools/android/VERSION_MACOSX_NDK b/tools/android/VERSION_MACOSX_NDK
index aac8005..bd19b05 100644
--- a/tools/android/VERSION_MACOSX_NDK
+++ b/tools/android/VERSION_MACOSX_NDK
@@ -1 +1 @@
-ed883dfaad6f27350eb3de426ccbf9abdc2943e8
+e8b6ecb5d15c4c4018a62b52aabc13e41b17df8f
diff --git a/tools/approve_results.dart b/tools/approve_results.dart
index d8adccd..616a27f 100755
--- a/tools/approve_results.dart
+++ b/tools/approve_results.dart
@@ -73,8 +73,30 @@
? loadResultsMap(path)
: <String, Map<String, dynamic>>{};
+/// Loads a log from logdog.
+Future<String> loadLog(String id, String step) async {
+ final logUrl = Uri.parse("https://logs.chromium.org/"
+ "logs/dart/buildbucket/cr-buildbucket.appspot.com/"
+ "$id/+/steps/$step?format=raw");
+ final client = new HttpClient();
+ final request =
+ await client.getUrl(logUrl).timeout(const Duration(seconds: 60));
+ final response = await request.close().timeout(const Duration(seconds: 60));
+ if (response.statusCode != HttpStatus.ok) {
+ throw new Exception("The log at $logUrl doesn't exist");
+ }
+ final contents = (await response
+ .transform(new Utf8Decoder())
+ .timeout(const Duration(seconds: 60))
+ .toList())
+ .join("");
+ client.close();
+ return contents;
+}
+
/// Loads the results from the bot.
-Future<List<Test>> loadResultsFromBot(String bot, ArgResults options) async {
+Future<List<Test>> loadResultsFromBot(String bot, ArgResults options,
+ Map<String, dynamic> changelistBuild) async {
if (options["verbose"]) {
print("Loading $bot...");
}
@@ -82,16 +104,28 @@
final tmpdir = await Directory.systemTemp.createTemp("approve_results.");
try {
// The 'latest' file contains the name of the latest build that we
- // should download.
- final build = await readFile(bot, "latest");
+ // should download. When preapproving a changelist, we instead find out
+ // which build the commit queue was rebased on.
+ final build = (changelistBuild != null
+ ? await loadLog(changelistBuild["id"],
+ "gsutil_find_latest_build/0/logs/raw_io.output_text_latest_/0")
+ : await readFile(bot, "latest"))
+ .trim();
// Asynchronously download the latest build and the current approved
- // results.
+ // results. Download try results from trybot try runs if preapproving.
+ final tryResults = <String, Map<String, dynamic>>{};
await Future.wait([
cpRecursiveGsutil(buildCloudPath(bot, build), tmpdir.path),
cpRecursiveGsutil(
"$approvedResultsStoragePath/$bot/approved_results.json",
"${tmpdir.path}/approved_results.json"),
+ new Future(() async {
+ if (changelistBuild != null) {
+ tryResults.addAll(parseResultsMap(await loadLog(
+ changelistBuild["id"], "test_results/0/logs/results.json/0")));
+ }
+ }),
]);
// Check the build was properly downloaded.
@@ -123,10 +157,58 @@
final result = results[key];
final approvedResult = approvedResults[key];
final flakiness = flaky[key];
+ // If preapproving results, allow new non-matching results that are
+ // different from the baseline. The approved results will be the current
+ // approved results, plus the difference between the tryrun's baseline and
+ // the tryrun's results.
+ if (tryResults.containsKey(key)) {
+ final tryResult = tryResults[key];
+ final wasFlake = flakiness != null &&
+ (flakiness["outcomes"] as List<dynamic>)
+ .contains(tryResult["result"]);
+ // Pick the try run result if the try result was not a flake and it's a
+ // non-matching result that's different than the approved result. If
+ // there is no approved result yet, use the latest result from the
+ // builder instead.
+ final baseResult = approvedResult ?? result;
+ if ((!wasFlake &&
+ !tryResult["matches"] &&
+ tryResult["result"] != result["result"])) {
+ // The approved_results.json format currently does not natively
+ // support preapproval, so preapproving turning one failure into
+ // another will turn the builder in question red until the CL lands.
+ if (!baseResult["matches"]) {
+ print("Warning: Preapproving changed failure modes will turn the "
+ "CI red until the CL is submitted: $bot: $key: "
+ "${baseResult["result"]} -> ${tryResult["result"]}");
+ }
+ result.clear();
+ result.addAll(tryResult);
+ } else {
+ if (approvedResult != null) {
+ result.clear();
+ result.addAll(approvedResult);
+ }
+ }
+ } else if (tryResults.isNotEmpty && approvedResult != null) {
+ result.clear();
+ result.addAll(approvedResult);
+ }
final name = result["name"];
final test = new Test(bot, name, result, approvedResult, flakiness);
tests.add(test);
}
+ // If preapproving and the CL has introduced new tests, add the new tests
+ // as well to the approved data.
+ final newTestKeys = new Set<String>.from(tryResults.keys)
+ .difference(new Set<String>.from(results.keys));
+ for (final key in newTestKeys) {
+ final result = tryResults[key];
+ final flakiness = flaky[key];
+ final name = result["name"];
+ final test = new Test(bot, name, result, null, flakiness);
+ tests.add(test);
+ }
if (options["verbose"]) {
print("Loaded $bot (${tests.length} tests).");
}
@@ -150,13 +232,18 @@
abbr: "n",
help: "Show changed results but don't approve.",
negatable: false);
+ parser.addOption("preapprove",
+ abbr: "p", help: "Preapprove the new failures in a gerrit CL.");
parser.addFlag("verbose",
abbr: "v", help: "Describe asynchronous operations.", negatable: false);
parser.addFlag("yes",
abbr: "y", help: "Approve the results.", negatable: false);
final options = parser.parse(args);
- if ((options["bot"].isEmpty && !options["list"]) || options["help"]) {
+ if ((options["preapprove"] == null &&
+ options["bot"].isEmpty &&
+ !options["list"]) ||
+ options["help"]) {
print("""
Usage: approve_results.dart [OPTION]...
List tests whose results are different from the previously approved results, and
@@ -218,12 +305,106 @@
return;
}
+ // Determine which builders have run for the changelist.
+ final changelistBuilds = <String, Map<String, dynamic>>{};
+ if (options["preapprove"] != null) {
+ if (options["verbose"]) {
+ print("Loading list of try runs...");
+ }
+ final gerritHost = "dart-review.googlesource.com";
+ final gerritProject = "sdk";
+ final prefix = "https://$gerritHost/c/$gerritProject/+/";
+ final gerrit = options["preapprove"];
+ if (!gerrit.startsWith(prefix)) {
+ stderr.writeln("error: $gerrit doesn't start with $prefix");
+ exitCode = 1;
+ return;
+ }
+ final components = gerrit.substring(prefix.length).split("/");
+ if (components.length != 2 ||
+ int.tryParse(components[0]) == null ||
+ int.tryParse(components[1]) == null) {
+ stderr.writeln("error: $gerrit must be in the form of "
+ "$prefix<changelist>/<patchset>");
+ exitCode = 1;
+ return;
+ }
+ final changelist = int.parse(components[0]);
+ final patchset = int.parse(components[1]);
+ final buildset = "buildset:patch/gerrit/$gerritHost/$changelist/$patchset";
+ final url = Uri.parse(
+ "https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search"
+ "?bucket=luci.dart.try"
+ "&tag=${Uri.encodeComponent(buildset)}"
+ "&fields=builds(id%2Ctags%2Cstatus%2Cstarted_ts)");
+ final client = new HttpClient();
+ final request =
+ await client.getUrl(url).timeout(const Duration(seconds: 30));
+ final response = await request.close().timeout(const Duration(seconds: 30));
+ if (response.statusCode != HttpStatus.ok) {
+ throw new Exception("Failed to request try runs for $gerrit");
+ }
+ final Map<String, dynamic> object = await response
+ .transform(new Utf8Decoder())
+ .transform(new JsonDecoder())
+ .first
+ .timeout(const Duration(seconds: 30));
+ client.close();
+ final builds = object["builds"];
+ if (builds == null) {
+ stderr.writeln(
+ "error: $prefix$changelist has no try runs for patchset $patchset");
+ exitCode = 1;
+ return;
+ }
+
+ // Prefer the newest completed build.
+ Map<String, dynamic> preferredBuild(
+ Map<String, dynamic> a, Map<String, dynamic> b) {
+ if (a != null && b == null) return a;
+ if (a == null && b != null) return b;
+ if (a != null && b != null) {
+ if (a["status"] == "COMPLETED" && b["status"] != "COMPLETED") return a;
+ if (a["status"] != "COMPLETED" && b["status"] == "COMPLETED") return b;
+ if (a["started_ts"] == null && b["started_ts"] != null) return a;
+ if (a["started_ts"] != null && b["started_ts"] == null) return b;
+ if (a["started_ts"] != null && b["started_ts"] != null) {
+ if (int.parse(a["started_ts"]) > int.parse(b["started_ts"])) return a;
+ if (int.parse(a["started_ts"]) < int.parse(b["started_ts"])) return b;
+ }
+ }
+ return b;
+ }
+
+ for (final build in builds) {
+ final tags = (build["tags"] as List<dynamic>).cast<String>();
+ final builder = tags
+ .firstWhere((tag) => tag.startsWith("builder:"))
+ .substring("builder:".length);
+ final ciBuilder = builder.replaceFirst(new RegExp("-try\$"), "");
+ if (!allBots.contains(ciBuilder)) {
+ continue;
+ }
+ changelistBuilds[ciBuilder] =
+ preferredBuild(changelistBuilds[ciBuilder], build);
+ }
+ if (options["verbose"]) {
+ print("Loaded list of try runs.");
+ }
+ }
+ final changelistBuilders = new Set<String>.from(changelistBuilds.keys);
+
// Select all the bots matching the glob patterns,
+ final finalBotList =
+ options["preapprove"] != null ? changelistBuilders : allBots;
+ final botPatterns = options["preapprove"] != null && options["bot"].isEmpty
+ ? ["*"]
+ : options["bot"];
final bots = new Set<String>();
- for (final botPattern in options["bot"]) {
+ for (final botPattern in botPatterns) {
final glob = new Glob(botPattern);
bool any = false;
- for (final bot in allBots) {
+ for (final bot in finalBotList) {
if (glob.matches(bot)) {
bots.add(bot);
any = true;
@@ -240,12 +421,28 @@
print("Selected bot: $bot");
}
+ // Error out if any of the requested try runs are incomplete.
+ bool anyIncomplete = false;
+ for (final bot in bots) {
+ if (options["preapprove"] != null &&
+ changelistBuilds[bot]["status"] != "COMPLETED") {
+ stderr.writeln("error: The try run for $bot isn't complete yet" +
+ changelistBuilds[bot]["status"]);
+ anyIncomplete = true;
+ }
+ }
+ if (anyIncomplete) {
+ exitCode = 1;
+ return;
+ }
+
// Load all the latest results for the selected bots, as well as flakiness
// data, and the set of currently approved results. Each bot's latest build
// is downloaded in parallel to make this phase faster.
final testListFutures = <Future>[];
for (final String bot in bots) {
- testListFutures.add(loadResultsFromBot(bot, options));
+ testListFutures
+ .add(loadResultsFromBot(bot, options, changelistBuilds[bot]));
}
// Collect all the tests from the synchronous downloads.
@@ -377,6 +574,10 @@
print("Note: Approving the failures will turn the "
"$botPlural green on the next commit.");
}
+ if (options["preapprove"] != null) {
+ print("Warning: Preapproval is currently not sticky and somebody else "
+ "approving before your CL has landed will undo your preapproval.");
+ }
while (true) {
stdout.write("Do you want to approve? (yes/no) [yes] ");
final line = stdin.readLineSync();
@@ -399,6 +600,18 @@
}
print("");
+ // Log who approved these results.
+ final username = Platform.environment["LOGNAME"] ??
+ Platform.environment["USER"] ??
+ Platform.environment["USERNAME"];
+ if (username == null || username == "") {
+ stderr.writeln("error: Your identity could not be established. "
+ "Please set one of the LOGNAME, USER, USERNAME environment variables.");
+ exitCode = 1;
+ return;
+ }
+ final now = new DateTime.now().toUtc().toIso8601String();
+
// Update approved_results.json for each bot with unapproved changes.
final outDirectory =
await Directory.systemTemp.createTemp("approved_results.");
@@ -414,10 +627,19 @@
print("Uploading approved results...");
final futures = <Future>[];
for (final String bot in unapprovedBots) {
- final testsList = testsForBots[bot];
+ Map<String, dynamic> approveData(Test test) {
+ final data = new Map<String, dynamic>.from(test.resultData);
+ if (!test.isApproved) {
+ data["approver"] = username;
+ data["approved_at"] = now;
+ }
+ return data;
+ }
+
+ final dataList = testsForBots[bot].map(approveData).toList();
final localPath = "${outDirectory.path}/$bot.json";
await new File(localPath).writeAsString(
- testsList.map((test) => jsonEncode(test.resultData) + "\n").join(""));
+ dataList.map((data) => jsonEncode(data) + "\n").join(""));
final remotePath =
"$approvedResultsStoragePath/$bot/approved_results.json";
futures.add(cpGsutil(localPath, remotePath)
diff --git a/tools/bots/compare_results.dart b/tools/bots/compare_results.dart
index e475e4c..f7c9efb 100755
--- a/tools/bots/compare_results.dart
+++ b/tools/bots/compare_results.dart
@@ -34,20 +34,23 @@
flaked = flakinessData != null &&
flakinessData["outcomes"].contains(map["result"]);
- String get key => "$name:$configuration";
+ String get key => "$configuration:$name";
}
class Event {
final Result before;
final Result after;
+ final Result approved;
- Event(this.before, this.after);
+ Event(this.before, this.after, this.approved);
bool get isNew => before == null;
bool get isNewPassing => before == null && after.matches;
bool get isNewFailing => before == null && !after.matches;
bool get changed => !unchanged;
bool get unchanged => before != null && before.outcome == after.outcome;
+ bool get isApproved => approved != null && approved.outcome == after.outcome;
+ bool get isUnapproved => !isApproved;
bool get remainedPassing => before.matches && after.matches;
bool get remainedFailing => !before.matches && !after.matches;
bool get flaked => after.flaked;
@@ -75,28 +78,41 @@
bool firstSection = true;
-bool search(String description, String searchFor, List<Event> events,
- ArgResults options) {
+bool search(
+ String description,
+ String searchForStatus,
+ String searchForApproval,
+ List<Event> events,
+ ArgResults options,
+ Map<String, Map<String, dynamic>> logs,
+ List<String> logSection) {
bool judgement = false;
bool beganSection = false;
int count = options["count"] != null ? int.parse(options["count"]) : null;
final configurations =
events.map((event) => event.after.configuration).toSet();
for (final event in events) {
- if (searchFor == "passing" &&
+ if (searchForStatus == "passing" &&
(event.after.flaked || !event.after.matches)) {
continue;
}
- if (searchFor == "flaky" && !event.after.flaked) {
+ if (searchForStatus == "flaky" && !event.after.flaked) {
continue;
}
- if (searchFor == "failing" && (event.after.flaked || event.after.matches)) {
+ if (searchForStatus == "failing" &&
+ (event.after.flaked || event.after.matches)) {
+ continue;
+ }
+ if (searchForApproval == "approved" && !event.isApproved) {
+ continue;
+ }
+ if (searchForApproval == "unapproved" && !event.isUnapproved) {
continue;
}
if (options["unchanged"] && !event.unchanged) continue;
if (options["changed"] && !event.changed) continue;
if (!beganSection) {
- if (options["human"]) {
+ if (options["human"] && !options["logs-only"]) {
if (!firstSection) {
print("");
}
@@ -124,30 +140,32 @@
break;
}
}
- if (options["human"]) {
- if (options["verbose"]) {
- String expected =
- after.matches ? "" : ", expected ${after.expectation}";
+ String output;
+ if (options["verbose"]) {
+ if (options["human"]) {
+ String expect = after.matches ? "" : ", expected ${after.expectation}";
if (before == null || before.outcome == after.outcome) {
- print("${name} ${event.description} "
- "(${event.after.outcome}${expected})");
+ output = "$name ${event.description} "
+ "(${event.after.outcome}${expect})";
} else {
- print("${name} ${event.description} "
- "(${event.before?.outcome} -> ${event.after.outcome}${expected})");
+ output = "$name ${event.description} "
+ "(${event.before?.outcome} -> ${event.after.outcome}${expect})";
}
} else {
- print(name);
- }
- } else {
- if (options["verbose"]) {
- print("$name "
- "${before?.outcome} ${after.outcome} "
+ output = "$name ${before?.outcome} ${after.outcome} "
"${before?.expectation} ${after.expectation} "
"${before?.matches} ${after.matches} "
- "${before?.flaked} ${after.flaked}");
- } else {
- print(name);
+ "${before?.flaked} ${after.flaked}";
}
+ } else {
+ output = name;
+ }
+ if (logs != null) {
+ final log = logs[event.after.key];
+ if (log != null) logSection?.add("\n\nLog for $output\n${log["log"]}");
+ }
+ if (!options["logs-only"]) {
+ print(output);
}
}
@@ -156,6 +174,8 @@
main(List<String> args) async {
final parser = new ArgParser();
+ parser.addFlag("approved",
+ abbr: 'A', negatable: false, help: "Show approved tests.");
parser.addFlag("changed",
abbr: 'c',
negatable: false,
@@ -180,6 +200,8 @@
negatable: false);
parser.addFlag("passing",
abbr: 'p', negatable: false, help: "Show passing tests.");
+ parser.addFlag("unapproved",
+ abbr: 'U', negatable: false, help: "Show unapproved tests.");
parser.addFlag("unchanged",
abbr: 'u',
negatable: false,
@@ -188,12 +210,18 @@
abbr: "v",
help: "Show the old and new result for each test",
negatable: false);
+ parser.addOption("logs",
+ abbr: "l", help: "Path to file holding logs of failing tests.");
+ parser.addFlag("logs-only",
+ help: "Only print logs of failing tests, no other output",
+ negatable: false);
final options = parser.parse(args);
if (options["help"]) {
print("""
-Usage: compare_results.dart [OPTION]... [BEFORE] [AFTER]
+Usage: compare_results.dart [OPTION]... BEFORE AFTER [APPROVED]
Compare the old and new test results and list tests that pass the filters.
+Three-way compare with the approved results if provided.
All tests are listed if no filters are given.
The options are as follows:
@@ -210,8 +238,9 @@
}
final parameters = options.rest;
- if (parameters.length != 2) {
- print("error: Expected two parameters (results before and results after)");
+ if (parameters.length != 2 && parameters.length != 3) {
+ print("error: Expected two or three parameters "
+ "(results before, results after, and (optionally) approved results)");
exitCode = 2;
return;
}
@@ -219,6 +248,12 @@
// Load the input and the flakiness data if specified.
final before = await loadResultsMap(parameters[0]);
final after = await loadResultsMap(parameters[1]);
+ final approved = 3 <= parameters.length
+ ? await loadResultsMap(parameters[2])
+ : <String, Map<String, dynamic>>{};
+ final logs = options['logs'] == null
+ ? <String, Map<String, dynamic>>{}
+ : await loadResultsMap(options['logs']);
final flakinessData = options["flakiness-data"] != null
? await loadResultsMap(options["flakiness-data"])
: <String, Map<String, dynamic>>{};
@@ -230,76 +265,112 @@
for (final name in names) {
final mapBefore = before[name];
final mapAfter = after[name];
+ final mapApproved = approved[name];
final resultBefore = mapBefore != null
? new Result.fromMap(mapBefore, flakinessData[name])
: null;
final resultAfter = new Result.fromMap(mapAfter, flakinessData[name]);
- final event = new Event(resultBefore, resultAfter);
+ final resultApproved = mapApproved != null
+ ? new Result.fromMap(mapApproved, flakinessData[name])
+ : null;
+ final event = new Event(resultBefore, resultAfter, resultApproved);
events.add(event);
}
+ final filterDescriptions = {
+ "passing": {
+ "unchanged": "continued to pass",
+ "changed": "began passing",
+ null: "passed",
+ },
+ "flaky": {
+ "unchanged": "are known to flake but didn't",
+ "changed": "flaked",
+ null: "are known to flake",
+ },
+ "failing": {
+ "unchanged": "continued to fail",
+ "changed": "began failing",
+ null: "failed",
+ },
+ null: {
+ "unchanged": "had the same result",
+ "changed": "changed result",
+ null: "ran",
+ },
+ };
+
+ final searchForStatuses =
+ ["passing", "flaky", "failing"].where((option) => options[option]);
+
+ final approvalDescriptions = {
+ "passing": {
+ "approved": " (approved)",
+ "unapproved": " (should be approved)",
+ null: "",
+ },
+ "flaky": {
+ "approved": " (approved result)",
+ "unapproved": " (unapproved result)",
+ null: "",
+ },
+ "failing": {
+ "approved": " (approved)",
+ "unapproved": " (needs approval)",
+ null: "",
+ },
+ null: {
+ "approved": " (approved)",
+ "unapproved": " (needs approval)",
+ null: "",
+ },
+ };
+
+ final searchForApprovals =
+ ["approved", "unapproved"].where((option) => options[option]);
+
// Report tests matching the filters.
+ final logSection = <String>[];
bool judgement = false;
- if (options["passing"] || options["flaky"] || options["failing"]) {
- if (options["passing"]) {
- String sectionHeader;
- if (options["unchanged"]) {
- sectionHeader = "The following tests continued to pass:";
- } else if (options["changed"]) {
- sectionHeader = "The following tests began passing:";
- } else {
- sectionHeader = "The following tests passed:";
+ for (final searchForStatus
+ in searchForStatuses.isNotEmpty ? searchForStatuses : <String>[null]) {
+ for (final searchForApproval in searchForApprovals.isNotEmpty
+ ? searchForApprovals
+ : <String>[null]) {
+ final searchForChanged = options["unchanged"]
+ ? "unchanged"
+ : options["changed"] ? "changed" : null;
+ final aboutStatus = filterDescriptions[searchForStatus][searchForChanged];
+ final aboutApproval =
+ approvalDescriptions[searchForStatus][searchForApproval];
+ final sectionHeader = "The following tests $aboutStatus$aboutApproval:";
+ final logSectionArg = searchForStatus == "failing" ? logSection : null;
+ bool possibleJudgement = search(sectionHeader, searchForStatus,
+ searchForApproval, events, options, logs, logSectionArg);
+ if ((searchForStatus == null || searchForStatus == "failing") &&
+ (searchForApproval == null || searchForApproval == "unapproved")) {
+ judgement = possibleJudgement;
}
- search(sectionHeader, "passing", events, options);
}
- if (options["flaky"]) {
- String sectionHeader;
- if (options["unchanged"]) {
- sectionHeader = "The following tests are known to flake but didn't:";
- } else if (options["changed"]) {
- sectionHeader = "The following tests flaked:";
- } else {
- sectionHeader = "The following tests are known to flake:";
- }
- search(sectionHeader, "flaky", events, options);
- }
- if (options["failing"]) {
- String sectionHeader;
- if (options["unchanged"]) {
- sectionHeader = "The following tests continued to fail:";
- } else if (options["changed"]) {
- sectionHeader = "The following tests began failing:";
- } else {
- sectionHeader = "The following tests failed:";
- }
- judgement = search(sectionHeader, "failing", events, options);
- }
- } else {
- String sectionHeader;
- if (options["unchanged"]) {
- sectionHeader = "The following tests had the same result:";
- } else if (options["changed"]) {
- sectionHeader = "The following tests changed result:";
- } else {
- sectionHeader = "The following tests ran:";
- }
- judgement = search(sectionHeader, null, events, options);
}
+ if (logSection.isNotEmpty) {
+ print(logSection.join());
+ }
// Exit 1 only if --judgement and any test failed.
if (options["judgement"]) {
- if (options["human"] && !firstSection) {
+ if (options["human"] && !options["logs-only"] && !firstSection) {
print("");
}
String oldNew =
options["unchanged"] ? "old " : options["changed"] ? "new " : "";
if (judgement) {
- if (options["human"]) {
+ if (options["human"] && !options["logs-only"]) {
print("There were ${oldNew}test failures.");
}
exitCode = 1;
} else {
- if (options["human"]) {
+ if (options["human"] && !options["logs-only"]) {
print("No ${oldNew}test failures were found.");
}
}
diff --git a/tools/bots/results.dart b/tools/bots/results.dart
index 36b27f2..8405a6d 100644
--- a/tools/bots/results.dart
+++ b/tools/bots/results.dart
@@ -115,12 +115,7 @@
Map<String, Map<String, dynamic>> createResultsMap(
List<Map<String, dynamic>> results) =>
- new Map<String, Map<String, dynamic>>.fromIterable(
- results
- // TODO: Temporarily discard results in the old flaky.json format
- // This can be removed once every bot has run once after this commit
- // has landed, purging all old flakiness information.
- .where((result) => result["configuration"] != null),
+ new Map<String, Map<String, dynamic>>.fromIterable(results,
key: (dynamic result) =>
"${result["configuration"]}:${result["name"]}");
diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json
index 70fdc8d..38bf229 100644
--- a/tools/bots/test_matrix.json
+++ b/tools/bots/test_matrix.json
@@ -98,6 +98,7 @@
"third_party/observatory_pub_packages/packages/",
"tools/sdks/dart-sdk/",
"pkg/async_helper/",
+ "pkg/build_integration/",
"pkg/dart_internal/",
"pkg/expect/",
"pkg/front_end/",
@@ -150,6 +151,7 @@
"tests/standalone/",
"tests/standalone_2/",
"pkg/async_helper/",
+ "pkg/build_integration/",
"pkg/dart_internal/",
"pkg/expect/",
"pkg/front_end/",
@@ -347,12 +349,6 @@
"compiler": "dart2analyzer",
"enable-asserts": true,
"use-sdk": true
- }},
- "analyzer-fasta_parser-linux": {
- "options": {
- "compiler": "dart2analyzer",
- "analyzer-use-fasta-parser": true,
- "use-sdk": true
}}
},
"builder_configurations": [
@@ -425,7 +421,6 @@
"name": "vm mixed mode tests",
"arguments": [
"-ndartkb-mixed-linux-${mode}-x64",
- "--compiler=dartkb",
"language_2",
"corelib_2",
"lib_2",
@@ -438,7 +433,6 @@
"name": "vm bytecode compiler tests",
"arguments": [
"-ndartkb-compile-linux-${mode}-x64",
- "--compiler=dartkb",
"language_2",
"corelib_2",
"lib_2",
@@ -451,7 +445,6 @@
"name": "vm interpreter tests",
"arguments": [
"-ndartkb-interpret-linux-${mode}-x64",
- "--compiler=dartkb",
"language_2",
"corelib_2",
"lib_2",
@@ -1325,13 +1318,17 @@
"steps": [
{
"name": "build dart",
- "script": "tools/build.py",
- "arguments": ["--arch=ia32,x64", "create_sdk" ]
+ "script": "tools/bots/dart_sdk.py",
+ "arguments": [],
+ "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-linux"}
+
},
{
- "name": "generate API docs",
+ "name": "build api docs",
"script": "tools/bots/dart_sdk.py",
- "arguments": [ "api_docs" ]
+ "arguments": [ "api_docs" ],
+ "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-linux"}
+
}
]
},
@@ -1343,8 +1340,10 @@
"steps": [
{
"name": "build dart",
- "script": "tools/build.py",
- "arguments": ["--arch=ia32,x64", "create_sdk" ]
+ "script": "tools/bots/dart_sdk.py",
+ "arguments": [],
+ "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-mac"}
+
}
]
},
@@ -1356,8 +1355,10 @@
"steps": [
{
"name": "build dart",
- "script": "tools/build.py",
- "arguments": ["--arch=ia32,x64", "create_sdk" ]
+ "script": "tools/bots/dart_sdk.py",
+ "arguments": [],
+ "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-win"}
+
}
]
},
@@ -1365,7 +1366,11 @@
"builders": [
"analyzer-linux-release",
"analyzer-mac-release",
- "analyzer-win-release"
+ "analyzer-win-release",
+ "analyzer-linux-release-analyzer",
+ "analyzer-linux-release-analyzer-new",
+ "analyzer-mac-release-analyzer",
+ "analyzer-win-release-analyzer"
],
"meta": {
"description": "This configuration is used by the analyzer builders."
@@ -1430,7 +1435,10 @@
]
},
{
- "builders": ["analyzer-analysis-server-linux"],
+ "builders": [
+ "analyzer-analysis-server-linux",
+ "analyzer-analysis-server-linux-analyzer"
+ ],
"meta": {
"description": "Analyze analyzer related packages."
},
@@ -1596,74 +1604,6 @@
]
},
{
- "builders": ["analyzer-use-fasta-parser-linux"],
- "meta": {
- "description": "Run the analyzer using the Fasta parser."
- },
- "steps": [
- {
- "name": "build dart",
- "script": "tools/build.py",
- "arguments": ["create_sdk"]
- },
- {
- "name": "analyze pkg/analyzer --use-fasta-parser",
- "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",
- "arguments": [
- "--use-fasta-parser",
- "pkg/analyzer"
- ]
- },
- {
- "name": "analyze pkg/analyzer --no-use-fasta-parser",
- "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",
- "arguments": [
- "--no-use-fasta-parser",
- "pkg/analyzer"
- ]
- },
- {
- "name": "analyze pkg/analyzer_cli --use-fasta-parser",
- "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",
- "arguments": [
- "--use-fasta-parser",
- "pkg/analyzer_cli"
- ]
- },
- {
- "name": "run language2 tests",
- "arguments": [
- "-nanalyzer-fasta_parser-linux",
- "language_2"
- ]
- },
- {
- "name": "analyze pkg/analyzer_cli --no-use-fasta-parser",
- "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",
- "arguments": [
- "--no-use-fasta-parser",
- "pkg/analyzer_cli"
- ]
- },
- {
- "name": "analyze pkg/analysis_server --use-fasta-parser",
- "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",
- "arguments": [
- "--use-fasta-parser",
- "pkg/analysis_server"
- ]
- },
- {
- "name": "analyze pkg/analysis_server --no-use-fasta-parser",
- "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",
- "arguments": [
- "--no-use-fasta-parser",
- "pkg/analysis_server"
- ]
- }
- ]
- },
- {
"builders": ["pkg-linux-release",
"pkg-win-release",
"pkg-mac-release"],
@@ -1795,7 +1735,7 @@
"--time",
"2700"
],
- "shards": 10,
+ "shards": 100,
"fileset": "fuzzer"
}
]
diff --git a/tools/bots/update_flakiness.dart b/tools/bots/update_flakiness.dart
index 87c584b..fb7130c 100755
--- a/tools/bots/update_flakiness.dart
+++ b/tools/bots/update_flakiness.dart
@@ -51,6 +51,16 @@
outcomes.add(result["result"]);
outcomes..sort();
}
+ if (testData["current"] == result["result"]) {
+ testData["current_counter"]++;
+ } else {
+ testData["current"] = result["result"];
+ testData["current_counter"] = 1;
+ }
+ final occurences =
+ testData.putIfAbsent("occurences", () => <String, dynamic>{});
+ occurences.putIfAbsent(result["result"], () => 0);
+ occurences[result["result"]]++;
}
}
@@ -63,6 +73,15 @@
for (final key in keys) {
final testData = data[key];
if (testData["outcomes"].length < 2) continue;
+ // Forgive tests that have become deterministic again. If they flake less
+ // than once in a 100 (p<1%), then if they flake again, the probability of
+ // them getting past 5 runs of deflaking is 1%^5 = 0.00000001%.
+ // TODO(sortie): Transitional compatibility until all flaky.json files have
+ // this new field.
+ if (testData["current_counter"] != null &&
+ 100 <= testData["current_counter"]) {
+ continue;
+ }
sink.writeln(jsonEncode(testData));
}
}
diff --git a/tools/buildtools/README.md b/tools/buildtools/README.md
index 8cd1cfb..7d79285 100644
--- a/tools/buildtools/README.md
+++ b/tools/buildtools/README.md
@@ -1,7 +1,7 @@
To build Dart for Mac and Linux, we pull Fuchsia's buildtools, which is also
-used by Flutter. Fuchsia's buildtools includes gn, ninja, and the clang
-toolchain. Fuchsia buildtools vends clang-format as part of the clang toolchain.
-Since Fuchsia's buildtools doesn't vend a clang toolchain for Windows, we can't
+used by Flutter. Fuchsia's buildtools includes gn, and the clang toolchain.
+Fuchsia buildtools vends clang-format as part of the clang toolchain. Since
+Fuchsia's buildtools doesn't vend a clang toolchain for Windows, we can't
get a Windows clang-format binary from it. Therefore, from Chromium's buildtools
here:
diff --git a/tools/buildtools/update.py b/tools/buildtools/update.py
index 5cae6a8..936e729 100755
--- a/tools/buildtools/update.py
+++ b/tools/buildtools/update.py
@@ -5,10 +5,10 @@
"""Pulls down tools required to build Dart."""
+import errno
import os
import platform
import subprocess
-import shutil
import sys
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -22,34 +22,6 @@
DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
-def Update():
- path = os.path.join(BUILDTOOLS, 'update.sh')
- command = ['/bin/bash', path, '--clang', '--gn']
- return subprocess.call(command, cwd=DART_ROOT)
-
-
-def UpdateGNOnWindows():
- sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'gn.exe.sha1')
- output_dir = os.path.join(BUILDTOOLS, 'win', 'gn.exe')
- downloader_script = os.path.join(
- DEPOT_PATH, 'download_from_google_storage.py')
- download_cmd = [
- 'python',
- downloader_script,
- '--no_auth',
- '--no_resume',
- '--quiet',
- '--platform=win*',
- '--bucket',
- 'chromium-gn',
- '-s',
- sha1_file,
- '-o',
- output_dir
- ]
- return subprocess.call(download_cmd)
-
-
def UpdateClangFormatOnWindows():
sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'clang-format.exe.sha1')
output_dir = os.path.join(BUILDTOOLS, 'win', 'clang-format.exe')
@@ -72,9 +44,20 @@
return subprocess.call(download_cmd)
-# On Mac and Linux we copy clang-format and gn to the place where git cl format
-# expects them to be.
-def CopyClangFormat():
+def CreateSymlink(symlink, link_name):
+ try:
+ os.symlink(symlink, link_name)
+ except OSError, e:
+ if e.errno == errno.EEXIST:
+ os.remove(link_name)
+ os.symlink(symlink, link_name)
+ else:
+ raise e
+
+
+# On Mac and Linux we symlink clang-format and gn to the place where
+# 'git cl format' expects them to be.
+def LinksForGitCLFormat():
if sys.platform == 'darwin':
platform = 'darwin'
tools = 'mac'
@@ -89,14 +72,14 @@
clang_format = os.path.join(
BUILDTOOLS, toolchain, 'clang', 'bin', 'clang-format')
- gn = os.path.join(BUILDTOOLS, toolchain, 'gn')
+ gn = os.path.join(BUILDTOOLS, 'gn')
dest_dir = os.path.join(BUILDTOOLS, tools)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
clang_format_dest = os.path.join(dest_dir, 'clang-format')
gn_dest = os.path.join(dest_dir, 'gn')
- shutil.copy2(clang_format, clang_format_dest)
- shutil.copy2(gn, gn_dest)
+ CreateSymlink(clang_format, clang_format_dest)
+ CreateSymlink(gn, gn_dest)
return 0
@@ -107,13 +90,8 @@
print('Not downloading buildtools binaries for ' + arch_id)
return 0
if sys.platform.startswith('win'):
- result = UpdateGNOnWindows()
- if result != 0:
- return result
return UpdateClangFormatOnWindows()
- if Update() != 0:
- return 1
- return CopyClangFormat()
+ return LinksForGitCLFormat()
if __name__ == '__main__':
diff --git a/tools/generate_compile_commands.py b/tools/generate_compile_commands.py
new file mode 100755
index 0000000..b94885d
--- /dev/null
+++ b/tools/generate_compile_commands.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import subprocess
+import json
+import utils
+
+### Python script to generate compile_commands.json and make it more useful.
+
+HOST_OS = utils.GuessOS()
+
+if not HOST_OS in ['linux', 'macos']:
+ print ("Generate compile_commands has not been ported to %s yet." % (HOST_OS))
+ exit
+
+if HOST_OS == 'linux':
+ ninja = 'ninja-linux64'
+ out_folder = ' out'
+if HOST_OS == 'macos':
+ ninja = 'ninja-mac'
+ out_folder = ' xcodebuild'
+
+subprocess.call("python tools/generate_buildfiles.py", shell=True)
+
+command_set = json.loads(
+ subprocess.check_output(
+ ninja + " -C " + out_folder + "/DebugX64 -t compdb cxx cc h", shell=True
+ )
+)
+
+commands = []
+for obj in command_set:
+ C = obj["command"]
+
+ # Skip precompiled mode, a lot of code is commented out in precompiled mode
+ if ("-DDART_PRECOMPILED_RUNTIME" in C):
+ continue
+
+ # Remove warnings
+ C = C.replace("-Werror", "")
+
+ obj["command"] = C
+ commands += [obj]
+
+json.dump(commands, open("compile_commands.json", "w"), indent=4)
diff --git a/tools/gn.py b/tools/gn.py
index 9b34862..bc05a61 100755
--- a/tools/gn.py
+++ b/tools/gn.py
@@ -496,16 +496,11 @@
starttime = time.time()
args = parse_args(argv)
- if sys.platform.startswith(('cygwin', 'win')):
- subdir = 'win'
- elif sys.platform == 'darwin':
- subdir = 'mac-x64'
- elif sys.platform.startswith('linux'):
- subdir = 'linux-x64'
- else:
- print 'Unknown platform: ' + sys.platform
+ gn = os.path.join(DART_ROOT, 'buildtools',
+ 'gn.exe' if utils.IsWindows() else 'gn')
+ if not os.path.isfile(gn):
+ print "Couldn't find the gn binary at path: " + gn
return 1
- gn = os.path.join(DART_ROOT, 'buildtools', subdir, 'gn')
commands = []
for target_os in args.os:
diff --git a/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.flutter.patch b/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.flutter.patch
index e4dcd66..7e5352b 100644
--- a/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.flutter.patch
+++ b/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.flutter.patch
@@ -1,5 +1,17 @@
+diff --git a/analysis_options.yaml b/analysis_options.yaml
+index 8957dfe03..68eea153e 100644
+--- a/analysis_options.yaml
++++ b/analysis_options.yaml
+@@ -114,7 +114,6 @@ linter:
+ # - parameter_assignments # we do this commonly
+ - prefer_adjacent_string_concatenation
+ - prefer_asserts_in_initializer_lists
+- - prefer_bool_in_asserts
+ - prefer_collection_literals
+ - prefer_conditional_assignment
+ - prefer_const_constructors
diff --git a/dev/devicelab/pubspec.yaml b/dev/devicelab/pubspec.yaml
-index 9dee6de4a12..8063988556d 100644
+index 9dee6de4a..806398855 100644
--- a/dev/devicelab/pubspec.yaml
+++ b/dev/devicelab/pubspec.yaml
@@ -5,7 +5,7 @@ homepage: https://github.com/flutter/flutter
@@ -12,7 +24,7 @@
dependencies:
args: 1.5.1
diff --git a/packages/flutter/pubspec.yaml b/packages/flutter/pubspec.yaml
-index 81564e6a2b4..9f85ed7ab4d 100644
+index 81564e6a2..9f85ed7ab 100644
--- a/packages/flutter/pubspec.yaml
+++ b/packages/flutter/pubspec.yaml
@@ -5,7 +5,7 @@ homepage: http://flutter.io
@@ -25,7 +37,7 @@
dependencies:
# To update these, use "flutter update-packages --force-upgrade".
diff --git a/packages/flutter_test/pubspec.yaml b/packages/flutter_test/pubspec.yaml
-index aa930cbdf49..531d24b7187 100644
+index aa930cbdf..531d24b71 100644
--- a/packages/flutter_test/pubspec.yaml
+++ b/packages/flutter_test/pubspec.yaml
@@ -2,7 +2,7 @@ name: flutter_test
@@ -38,7 +50,7 @@
dependencies:
# To update these, use "flutter update-packages --force-upgrade".
diff --git a/packages/flutter_tools/pubspec.yaml b/packages/flutter_tools/pubspec.yaml
-index 5f89d0f18ea..a869162b0e8 100644
+index 5f89d0f18..a869162b0 100644
--- a/packages/flutter_tools/pubspec.yaml
+++ b/packages/flutter_tools/pubspec.yaml
@@ -5,7 +5,7 @@ author: Flutter Authors <flutter-dev@googlegroups.com>
diff --git a/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.patch b/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.patch
index 0d76138..4a016f9 100644
--- a/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.patch
+++ b/tools/patches/flutter-engine/f9ebf2129732fd2b606286fdf58e500384b8a0bc.patch
@@ -1,5 +1,5 @@
diff --git a/DEPS b/DEPS
-index bb8bca813..701871f79 100644
+index cd4acaf5e..28f8d1b72 100644
--- a/DEPS
+++ b/DEPS
@@ -47,7 +47,7 @@ vars = {
@@ -11,6 +11,15 @@
'dart_dart_style_tag': '1.2.0',
'dart_dartdoc_tag': 'v0.24.1',
'dart_fixnum_tag': '0.10.8',
+@@ -60,7 +60,7 @@ vars = {
+ 'dart_http_throttle_tag': '1.0.2',
+ 'dart_intl_tag': '0.15.7',
+ 'dart_json_rpc_2_tag': '2.0.9',
+- 'dart_linter_tag': '0.1.71',
++ 'dart_linter_tag': '0.1.75',
+ 'dart_logging_tag': '0.11.3+2',
+ 'dart_markdown_tag': '2.0.2',
+ 'dart_matcher_tag': '0.12.3',
diff --git a/lib/snapshot/BUILD.gn b/lib/snapshot/BUILD.gn
index ef06063a2..fa18ebfbf 100644
--- a/lib/snapshot/BUILD.gn
diff --git a/tools/test.dart b/tools/test.dart
index 5914e5f..abb3b35 100755
--- a/tools/test.dart
+++ b/tools/test.dart
@@ -15,6 +15,67 @@
const int deflakingCount = 5;
+/// Quotes a string in shell single quote mode. This function produces a single
+/// shell argument that evaluates to the exact string provided, handling any
+/// special characters in the input string. Shell single quote mode works uses
+/// the single quote character as the delimiter and uses the characters
+/// in-between verbatim without any special processing. To insert the single
+/// quote character itself, escape single quote mode, insert an escaped single
+/// quote, and then return to single quote mode.
+///
+/// Examples:
+/// foo becomes 'foo'
+/// foo bar becomes 'foo bar'
+/// foo\ bar becomes 'foo\ bar'
+/// foo's bar becomes 'foo '\''s bar'
+/// foo "b"ar becomes 'foo "b"'
+/// foo
+/// bar becomes 'foo
+/// bar'
+String shellSingleQuote(String string) {
+ return "'${string.replaceAll("'", "'\\''")}'";
+}
+
+/// Like [shellSingleQuote], but if the string only contains safe ASCII
+/// characters, don't quote it. Note that it's not always safe to omit the
+/// quotes even if the string only has safe characters, as doing so might match
+/// a shell keyword or a shell builtin in the first argument in a command. It
+/// should be safe to use this for the second argument onwards in a command.
+String simpleShellSingleQuote(String string) {
+ return new RegExp(r"^[a-zA-Z0-9%+,./:_-]*$").hasMatch(string)
+ ? string
+ : shellSingleQuote(string);
+}
+
+/// Runs a process and exits likewise if the process exits non-zero.
+Future<ProcessResult> runProcess(
+ String executable, List<String> arguments) async {
+ final processResult = await Process.run(executable, arguments);
+ if (processResult.exitCode != 0) {
+ final command =
+ ([executable]..addAll(arguments)).map(simpleShellSingleQuote).join(" ");
+ throw new Exception("Command exited ${processResult.exitCode}: $command\n"
+ "${processResult.stdout}\n${processResult.stderr}");
+ }
+ return processResult;
+}
+
+/// Runs a process and exits likewise if the process exits non-zero, but let the
+/// child process inherit out stdio handles.
+Future<ProcessResult> runProcessInheritStdio(
+ String executable, List<String> arguments) async {
+ final process = await Process.start(executable, arguments,
+ mode: ProcessStartMode.inheritStdio);
+ final exitCode = await process.exitCode;
+ final processResult = new ProcessResult(process.pid, exitCode, "", "");
+ if (processResult.exitCode != 0) {
+ final command =
+ ([executable]..addAll(arguments)).map(simpleShellSingleQuote).join(" ");
+ throw new Exception("Command exited ${processResult.exitCode}: $command");
+ }
+ return processResult;
+}
+
/// Returns the operating system of a builder.
String systemOfBuilder(String builder) {
return builder.split("-").firstWhere(
@@ -188,6 +249,7 @@
// Run each step like the builder would, deflaking tests that need it.
final stepResultsPaths = <String>[];
+ final stepLogsPaths = <String>[];
for (int stepIndex = 0; stepIndex < testSteps.length; stepIndex++) {
// Run the test step.
final testStep = testSteps[stepIndex];
@@ -200,22 +262,29 @@
.cast<String>();
final fullArguments = <String>[]
..addAll(stepArguments)
- ..addAll(
- ["--output-directory=${stepDirectory.path}", "--write-results"])
+ ..addAll([
+ "--output-directory=${stepDirectory.path}",
+ "--clean-exit",
+ "--silent-failures",
+ "--write-results",
+ "--write-logs",
+ ])
..addAll(options.rest);
+ print("".padLeft(80, "="));
print("$stepName: Running tests");
- final testProcess = await Process.start("tools/test.py", fullArguments);
- await testProcess.exitCode;
+ print("".padLeft(80, "="));
+ await runProcessInheritStdio("tools/test.py", fullArguments);
stepResultsPaths.add("${stepDirectory.path}/results.json");
+ stepLogsPaths.add("${stepDirectory.path}/logs.json");
// Find the list of tests to deflake.
- final deflakeListOutput = await Process.run(Platform.resolvedExecutable, [
+ final deflakeListOutput = await runProcess(Platform.resolvedExecutable, [
"tools/bots/compare_results.dart",
"--changed",
"--failing",
"--passing",
"--flakiness-data=${outDirectory.path}/flaky.json",
"${outDirectory.path}/previous.json",
- "${stepDirectory.path}/results.json"
+ "${stepDirectory.path}/results.json",
]);
final deflakeListPath = "${stepDirectory.path}/deflake.list";
final deflakeListFile = new File(deflakeListPath);
@@ -225,27 +294,30 @@
for (int i = 1;
deflakeListOutput.stdout != "" && i <= deflakingCount;
i++) {
+ print("".padLeft(80, "="));
print("$stepName: Running deflaking iteration $i");
+ print("".padLeft(80, "="));
final deflakeDirectory = new Directory("${stepDirectory.path}/$i");
await deflakeDirectory.create();
final deflakeArguments = <String>[]
..addAll(stepArguments)
..addAll([
"--output-directory=${deflakeDirectory.path}",
+ "--clean-exit",
+ "--silent-failures",
"--write-results",
- "--test-list=$deflakeListPath"
+ "--test-list=$deflakeListPath",
])
..addAll(options.rest);
- final deflakeProcess =
- await Process.start("tools/test.py", deflakeArguments);
- await deflakeProcess.exitCode;
+ await runProcessInheritStdio("tools/test.py", deflakeArguments);
deflakingResultsPaths.add("${deflakeDirectory.path}/results.json");
}
// Update the flakiness information based on what we've learned.
print("$stepName: Updating flakiness information");
- await Process.run(
- "tools/bots/update_flakiness.dart",
+ await runProcess(
+ Platform.resolvedExecutable,
[
+ "tools/bots/update_flakiness.dart",
"--input=${outDirectory.path}/flaky.json",
"--output=${outDirectory.path}/flaky.json",
"${stepDirectory.path}/results.json",
@@ -256,19 +328,31 @@
stepResultsPaths
.map((path) => new File(path).readAsStringSync())
.join(""));
+ // Collect all the logs from all the steps.
+ await new File("${outDirectory.path}/logs.json").writeAsString(stepLogsPaths
+ .map((path) => new File(path).readAsStringSync())
+ .join(""));
// Write out the final comparison.
- print("");
- final compareOutput = await Process.run("tools/bots/compare_results.dart", [
+ print("".padLeft(80, "="));
+ print("Test Results");
+ print("".padLeft(80, "="));
+ final compareOutput = await runProcess(Platform.resolvedExecutable, [
+ "tools/bots/compare_results.dart",
"--human",
"--verbose",
"--changed",
"--failing",
"--passing",
"--flakiness-data=${outDirectory.path}/flaky.json",
+ "--logs=${outDirectory.path}/logs.json",
"${outDirectory.path}/previous.json",
- "${outDirectory.path}/results.json"
+ "${outDirectory.path}/results.json",
]);
- stdout.write(compareOutput.stdout);
+ if (compareOutput.stdout == "") {
+ print("There were no test failures.");
+ } else {
+ stdout.write(compareOutput.stdout);
+ }
} finally {
await outDirectory.delete(recursive: true);
}
diff --git a/tools/testing/dart/configuration.dart b/tools/testing/dart/configuration.dart
index 53f95df..aa0b4f1 100644
--- a/tools/testing/dart/configuration.dart
+++ b/tools/testing/dart/configuration.dart
@@ -33,7 +33,8 @@
this.isVerbose,
this.listTests,
this.listStatusFiles,
- this.noStatus,
+ this.cleanExit,
+ this.silentFailures,
this.printTiming,
this.printReport,
this.reportInJson,
@@ -58,6 +59,7 @@
this.testServerCrossOriginPort,
this.testDriverErrorPort,
this.localIP,
+ this.keepGeneratedFiles,
this.dart2jsOptions,
String packages,
this.packageRoot,
@@ -83,7 +85,8 @@
final bool isVerbose;
final bool listTests;
final bool listStatusFiles;
- final bool noStatus;
+ final bool cleanExit;
+ final bool silentFailures;
final bool printTiming;
final bool printReport;
final bool reportInJson;
@@ -112,7 +115,6 @@
bool get useAnalyzerCfe => configuration.useAnalyzerCfe;
bool get useAnalyzerFastaParser => configuration.useAnalyzerFastaParser;
bool get useBlobs => configuration.useBlobs;
- bool get keepGeneratedFiles => configuration.keepGeneratedFiles;
bool get useSdk => configuration.useSdk;
bool get useFastStartup => configuration.useFastStartup;
bool get useEnableAsserts => configuration.enableAsserts;
@@ -140,6 +142,7 @@
final int testServerCrossOriginPort;
final int testDriverErrorPort;
final String localIP;
+ final bool keepGeneratedFiles;
/// Extra dart2js options passed to the testing script.
final List<String> dart2jsOptions;
diff --git a/tools/testing/dart/options.dart b/tools/testing/dart/options.dart
index 3bea2d7..72ef8d4 100644
--- a/tools/testing/dart/options.dart
+++ b/tools/testing/dart/options.dart
@@ -226,10 +226,12 @@
'List status files for test-suites. Do not run any test suites.',
hide: true),
new _Option.bool(
- 'no_status',
- 'Do not use status files for test expectations. Use Skip '
- 'and Slow from status files. Return exit code 0 if tests '
- 'run and return results.',
+ 'clean_exit', 'Exit 0 if tests ran and results were output.',
+ hide: true),
+ new _Option.bool(
+ 'silent_failures',
+ "Don't complain about failing tests. This is useful when in "
+ "combination with --write-results.",
hide: true),
new _Option.bool('report_in_json',
'When listing with --list, output result summary in JSON.',
@@ -333,14 +335,15 @@
/// options.
static final _blacklistedOptions = [
'build_directory',
- 'debug_output_directory',
'chrome',
+ 'clean_exit',
'copy_coredumps',
'dart',
- 'flutter',
+ 'debug_output_directory',
'drt',
'exclude_suite',
'firefox',
+ 'flutter',
'local_ip',
'output_directory',
'progress',
@@ -348,15 +351,25 @@
'safari',
'shard',
'shards',
+ 'silent_failures',
'step_name',
'tasks',
'time',
'verbose',
- 'write_logs',
'write_debug_log',
+ 'write_logs',
'write_results',
].toSet();
+ /// The set of objects which the named configuration should imply.
+ static final _namedConfigurationOptions = [
+ 'system',
+ 'arch',
+ 'mode',
+ 'compiler',
+ 'runtime',
+ ].toSet();
+
/// Parses a list of strings as test options.
///
/// Returns a list of configurations in which to run the tests.
@@ -473,6 +486,18 @@
}
}
+ // If a named configuration was specified ensure no other options, which are
+ // implied by the named configuration, were specified.
+ if (configuration['named_configuration'] is String) {
+ for (final optionName in _namedConfigurationOptions) {
+ if (configuration.containsKey(optionName)) {
+ final namedConfig = configuration['named_configuration'];
+ _fail("The named configuration '$namedConfig' implies "
+ "'$optionName'. Try removing '$optionName'.");
+ }
+ }
+ }
+
// Apply default values for unspecified options.
for (var option in _options) {
if (!configuration.containsKey(option.name)) {
@@ -497,12 +522,16 @@
/// Given a set of parsed option values, returns the list of command line
/// arguments that would reproduce that configuration.
- List<String> _reproducingCommand(Map<String, dynamic> data) {
+ List<String> _reproducingCommand(
+ Map<String, dynamic> data, bool usingNamedConfiguration) {
var arguments = <String>[];
for (var option in _options) {
var name = option.name;
- if (!data.containsKey(name) || _blacklistedOptions.contains(name)) {
+ if (!data.containsKey(name) ||
+ _blacklistedOptions.contains(name) ||
+ (usingNamedConfiguration &&
+ _namedConfigurationOptions.contains(name))) {
continue;
}
@@ -634,7 +663,7 @@
// Expand compilers.
for (var compiler in compilers) {
// Expand modes.
- String modes = data["mode"] ?? compiler.defaultMode.name;
+ String modes = (data["mode"] as String) ?? compiler.defaultMode.name;
if (modes == "all") modes = "debug,release,product";
for (var modeName in modes.split(",")) {
var mode = Mode.find(modeName);
@@ -650,7 +679,6 @@
useAnalyzerFastaParser:
data["analyzer_use_fasta_parser"] as bool,
useBlobs: data["use_blobs"] as bool,
- keepGeneratedFiles: data["keep_generated_files"] as bool,
useSdk: data["use_sdk"] as bool,
useFastStartup: data["fast_startup"] as bool,
useDart2JSWithKernel: data["dart2js_with_kernel"] as bool,
@@ -676,13 +704,15 @@
isVerbose: data["verbose"] as bool,
listTests: data["list"] as bool,
listStatusFiles: data["list_status_files"] as bool,
- noStatus: data["no_status"] as bool,
+ cleanExit: data["clean_exit"] as bool,
+ silentFailures: data["silent_failures"] as bool,
printTiming: data["time"] as bool,
printReport: data["report"] as bool,
reportInJson: data["report_in_json"] as bool,
resetBrowser: data["reset_browser_configuration"] as bool,
skipCompilation: data["skip_compilation"] as bool,
- useKernelBytecode: compiler == Compiler.dartkb,
+ useKernelBytecode:
+ innerConfiguration.compiler == Compiler.dartkb,
writeDebugLog: data["write_debug_log"] as bool,
writeResults: data["write_results"] as bool,
writeLogs: data["write_logs"] as bool,
@@ -693,6 +723,7 @@
dartPath: data["dart"] as String,
dartPrecompiledPath: data["dart_precompiled"] as String,
flutterPath: data["flutter"] as String,
+ keepGeneratedFiles: data["keep_generated_files"] as bool,
taskCount: data["tasks"] as int,
shardCount: data["shards"] as int,
shard: data["shard"] as int,
@@ -707,7 +738,8 @@
packageRoot: data["package_root"] as String,
suiteDirectory: data["suite_dir"] as String,
outputDirectory: data["output_directory"] as String,
- reproducingArguments: _reproducingCommand(data),
+ reproducingArguments:
+ _reproducingCommand(data, namedConfiguration != null),
fastTestsOnly: data["fast_tests"] as bool,
printPassingStdout: data["print_passing_stdout"] as bool);
diff --git a/tools/testing/dart/runtime_configuration.dart b/tools/testing/dart/runtime_configuration.dart
index ff8efbb..8330b51 100644
--- a/tools/testing/dart/runtime_configuration.dart
+++ b/tools/testing/dart/runtime_configuration.dart
@@ -81,7 +81,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
// TODO(ahe): Make this method abstract.
throw "Unimplemented runtime '$runtimeType'";
}
@@ -99,7 +100,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
return <Command>[];
}
}
@@ -125,7 +127,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
// TODO(ahe): Avoid duplication of this method between d8 and jsshell.
checkArtifact(artifact);
return [
@@ -147,7 +150,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
checkArtifact(artifact);
return [
Command.jsCommandLine(
@@ -205,7 +209,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
String script = artifact.filename;
String type = artifact.mimeType;
if (script != null &&
@@ -215,7 +220,9 @@
type != 'application/kernel-ir-fully-linked') {
throw "Dart VM cannot run files of type '$type'.";
}
-
+ if (isCrashExpected) {
+ arguments.insert(0, '--suppress-core-dump');
+ }
String executable = suite.dartVmBinaryFileName;
if (type == 'application/kernel-ir-fully-linked') {
executable = suite.dartVmExecutableFileName;
@@ -230,7 +237,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
String script = artifact.filename;
String type = artifact.mimeType;
if (script != null &&
@@ -252,7 +260,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
String script = artifact.filename;
String type = artifact.mimeType;
if (script != null && type != 'application/dart-precompiled') {
@@ -279,7 +288,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
String script = artifact.filename;
String type = artifact.mimeType;
if (script != null && type != 'application/dart-precompiled') {
@@ -315,7 +325,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
String executable = suite.dartVmBinaryFileName;
return selfCheckers
.map((String tester) => Command.vmBatch(
@@ -336,7 +347,8 @@
TestSuite suite,
CommandArtifact artifact,
List<String> arguments,
- Map<String, String> environmentOverrides) {
+ Map<String, String> environmentOverrides,
+ bool isCrashExpected) {
throw "Unimplemented runtime '$runtimeType'";
}
}
diff --git a/tools/testing/dart/test_configurations.dart b/tools/testing/dart/test_configurations.dart
index 00e5ebf..d20f969 100644
--- a/tools/testing/dart/test_configurations.dart
+++ b/tools/testing/dart/test_configurations.dart
@@ -188,6 +188,9 @@
printFailures = false;
eventListener.add(new StatusFileUpdatePrinter());
}
+ if (firstConf.silentFailures) {
+ printFailures = false;
+ }
eventListener.add(new SummaryPrinter());
if (printFailures) {
// The buildbot has it's own failure summary since it needs to wrap it
@@ -222,7 +225,7 @@
if (listTests) {
eventListener.add(new SummaryPrinter(jsonOnly: reportInJson));
} else {
- if (!firstConf.noStatus) {
+ if (!firstConf.cleanExit) {
eventListener.add(new ExitCodeSetter());
}
eventListener.add(new IgnoredTestMonitor());
diff --git a/tools/testing/dart/test_suite.dart b/tools/testing/dart/test_suite.dart
index 4a76a66..cb2ca1e 100644
--- a/tools/testing/dart/test_suite.dart
+++ b/tools/testing/dart/test_suite.dart
@@ -483,6 +483,9 @@
}
void _addTest(ExpectationSet testExpectations, String testName) {
+ var fullName = 'cc/$testName';
+ var expectations = testExpectations.expectations(fullName);
+
var args = configuration.standardOptions.toList();
if (configuration.compilerConfiguration.previewDart2) {
final dfePath = new Path("$buildDir/gen/kernel-service.dart.snapshot")
@@ -491,13 +494,14 @@
// '--dfe' has to be the first argument for run_vm_test to pick it up.
args.insert(0, '--dfe=$dfePath');
}
+ if (expectations.contains(Expectation.crash)) {
+ args.insert(0, '--suppress-core-dump');
+ }
args.add(testName);
var command = Command.process(
'run_vm_unittest', targetRunnerPath, args, environmentOverrides);
- var fullName = 'cc/$testName';
- var expectations = testExpectations.expectations(fullName);
enqueueNewTestCase(fullName, [command], expectations);
}
@@ -828,15 +832,16 @@
allVmOptions = vmOptions.toList()..addAll(extraVmOptions);
}
- var commands =
- makeCommands(info, vmOptionsVariant, allVmOptions, commonArguments);
var expectations = testExpectations.expectations(testName);
+ var isCrashExpected = expectations.contains(Expectation.crash);
+ var commands = makeCommands(info, vmOptionsVariant, allVmOptions,
+ commonArguments, isCrashExpected);
enqueueNewTestCase(testName, commands, expectations, info);
}
}
List<Command> makeCommands(TestInformation info, int vmOptionsVariant,
- List<String> vmOptions, List<String> args) {
+ List<String> vmOptions, List<String> args, bool isCrashExpected) {
var commands = <Command>[];
var compilerConfiguration = configuration.compilerConfiguration;
var sharedOptions = info.optionsFromFile['sharedOptions'] as List<String>;
@@ -891,14 +896,14 @@
compilationArtifact);
Map<String, String> environment = environmentOverrides;
- Map<String, String> extraEnv = info.optionsFromFile['environment'];
+ var extraEnv = info.optionsFromFile['environment'] as Map<String, String>;
if (extraEnv != null) {
environment = new Map.from(environment)..addAll(extraEnv);
}
return commands
- ..addAll(configuration.runtimeConfiguration.computeRuntimeCommands(
- this, compilationArtifact, runtimeArguments, environment));
+ ..addAll(configuration.runtimeConfiguration.computeRuntimeCommands(this,
+ compilationArtifact, runtimeArguments, environment, isCrashExpected));
}
CreateTest makeTestCaseCreator(Map<String, dynamic> optionsFromFile) {
