Display flaky failure logs on try jobs
The step "find ignored flaky test failure logs" was being run on
CI builders but not on try jobs. Run this step on try jobs,
so flakiness on try jobs can be investigated better.
Bug: https://github.com/dart-lang/dart_ci/issues/160
Change-Id: I57b8466c4ba255528cdfed3c196a8aee12e79058
Reviewed-on: https://dart-review.googlesource.com/c/recipes/+/357040
Reviewed-by: Alexander Thomas <athom@google.com>
Commit-Queue: William Hesse <whesse@google.com>
diff --git a/recipe_modules/dart/api.py b/recipe_modules/dart/api.py
index ce5c6c1..3daea89 100644
--- a/recipe_modules/dart/api.py
+++ b/recipe_modules/dart/api.py
@@ -700,14 +700,13 @@
'find tests that began passing',
args + ["--changed", "--passing"],
stdout=self.m.raw_io.output_text(add_output_log=True)).stdout
+ links["ignored flaky test failure logs"] = self.m.step(
+ 'find ignored flaky test failure logs',
+ args + args_logs + ["--flaky"],
+ stdout=self.m.raw_io.output_text(add_output_log=True)).stdout
judgement_args += ["--changed", "--failing"]
if self._try_builder(): # pragma: no cover
judgement_args += ["--passing"]
- else:
- links["ignored flaky test failure logs"] = self.m.step(
- 'find ignored flaky test failure logs',
- args + args_logs + ["--flaky"],
- stdout=self.m.raw_io.output_text(add_output_log=True)).stdout
# Run the results step that determines the status of the build.
if not self._supports_result_database() or self._is_bisecting():
diff --git a/recipe_modules/dart/examples/example.expected/branch-try-builder-does-not-publish-results.json b/recipe_modules/dart/examples/example.expected/branch-try-builder-does-not-publish-results.json
index 483e652..9a08a42 100644
--- a/recipe_modules/dart/examples/example.expected/branch-try-builder-does-not-publish-results.json
+++ b/recipe_modules/dart/examples/example.expected/branch-try-builder-does-not-publish-results.json
@@ -2692,6 +2692,40 @@
"--verbose",
"[CACHE]/builder/sdk/LATEST/results.json",
"{\"name\":\"co19/Language/Classes/Abstract_Instance_Members/inherited_t01\",\"configuration\":\"dartk-linux-product-x64\",\"suite\":\"co19\",\"test_name\":\"Language/Classes/Abstract_Instance_Members/inherited_t01\",\"time_ms\":451,\"result\":\"CompileTimeError\",\"expected\":\"CompileTimeError\",\"matches\":true,\"commit_time\":1551185312,\"commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"build_number\":\"2404\",\"builder_name\":\"vm-kernel-linux-product-x64\",\"bot_name\":\"trusty-dart-68765ebb-us-central1-b-2ls0\",\"flaky\":false,\"previous_flaky\":false,\"previous_result\":\"CompileTimeError\",\"previous_commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"previous_commit_time\":1551185312,\"previous_build_number\":2403,\"changed\":false}\n{\"name\":\"co19/Language/Classes/Abstract_Instance_Members/inherited_t02\",\"configuration\":\"dartk-linux-product-x64\",\"suite\":\"co19\",\"test_name\":\"Language/Classes/Abstract_Instance_Members/inherited_t02\",\"time_ms\":496,\"result\":\"CompileTimeError\",\"expected\":\"CompileTimeError\",\"matches\":true,\"commit_time\":1551185312,\"commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"build_number\":\"2404\",\"builder_name\":\"vm-kernel-linux-product-x64\",\"bot_name\":\"trusty-dart-68765ebb-us-central1-b-2ls0\",\"flaky\":false,\"previous_flaky\":false,\"previous_result\":\"CompileTimeError\",\"previous_commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"previous_commit_time\":1551185312,\"previous_build_number\":2403,\"changed\":false}\n",
+ "--logs",
+ "{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"name\": \"Flaky/Test/1\", \"configuration\": \"example_configuration\"}\n{\"name\": \"Flaky/Test/2\", \"configuration\": \"example_configuration\"}\n{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"name\": \"Flaky/Test/1\", \"configuration\": \"example_configuration\"}\n{\"name\": \"Flaky/Test/2\", \"configuration\": \"example_configuration\"}\n",
+ "--logs-only",
+ "--flaky"
+ ],
+ "cwd": "[CACHE]/builder/sdk",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "find ignored flaky test failure logs",
+ "~followup_annotations": [
+ "@@@STEP_LOG_END@raw_io.output_text@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/builder/sdk/tools/sdks/dart-sdk/bin/dart",
+ "[CACHE]/builder/sdk/tools/bots/compare_results.dart",
+ "--flakiness-data",
+ "",
+ "--human",
+ "--verbose",
+ "[CACHE]/builder/sdk/LATEST/results.json",
+ "{\"name\":\"co19/Language/Classes/Abstract_Instance_Members/inherited_t01\",\"configuration\":\"dartk-linux-product-x64\",\"suite\":\"co19\",\"test_name\":\"Language/Classes/Abstract_Instance_Members/inherited_t01\",\"time_ms\":451,\"result\":\"CompileTimeError\",\"expected\":\"CompileTimeError\",\"matches\":true,\"commit_time\":1551185312,\"commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"build_number\":\"2404\",\"builder_name\":\"vm-kernel-linux-product-x64\",\"bot_name\":\"trusty-dart-68765ebb-us-central1-b-2ls0\",\"flaky\":false,\"previous_flaky\":false,\"previous_result\":\"CompileTimeError\",\"previous_commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"previous_commit_time\":1551185312,\"previous_build_number\":2403,\"changed\":false}\n{\"name\":\"co19/Language/Classes/Abstract_Instance_Members/inherited_t02\",\"configuration\":\"dartk-linux-product-x64\",\"suite\":\"co19\",\"test_name\":\"Language/Classes/Abstract_Instance_Members/inherited_t02\",\"time_ms\":496,\"result\":\"CompileTimeError\",\"expected\":\"CompileTimeError\",\"matches\":true,\"commit_time\":1551185312,\"commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"build_number\":\"2404\",\"builder_name\":\"vm-kernel-linux-product-x64\",\"bot_name\":\"trusty-dart-68765ebb-us-central1-b-2ls0\",\"flaky\":false,\"previous_flaky\":false,\"previous_result\":\"CompileTimeError\",\"previous_commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"previous_commit_time\":1551185312,\"previous_build_number\":2403,\"changed\":false}\n",
"--judgement",
"--changed",
"--failing",
diff --git a/recipe_modules/dart/examples/example.expected/try-builder-publishes-results.json b/recipe_modules/dart/examples/example.expected/try-builder-publishes-results.json
index 3791662..91dab71 100644
--- a/recipe_modules/dart/examples/example.expected/try-builder-publishes-results.json
+++ b/recipe_modules/dart/examples/example.expected/try-builder-publishes-results.json
@@ -2765,6 +2765,40 @@
},
{
"cmd": [
+ "[CACHE]/builder/sdk/tools/sdks/dart-sdk/bin/dart",
+ "[CACHE]/builder/sdk/tools/bots/compare_results.dart",
+ "--flakiness-data",
+ "",
+ "--human",
+ "--verbose",
+ "[CACHE]/builder/sdk/LATEST/results.json",
+ "{\"name\":\"co19/Language/Classes/Abstract_Instance_Members/inherited_t01\",\"configuration\":\"dartk-linux-product-x64\",\"suite\":\"co19\",\"test_name\":\"Language/Classes/Abstract_Instance_Members/inherited_t01\",\"time_ms\":451,\"result\":\"CompileTimeError\",\"expected\":\"CompileTimeError\",\"matches\":true,\"commit_time\":1551185312,\"commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"build_number\":\"2404\",\"builder_name\":\"vm-kernel-linux-product-x64\",\"bot_name\":\"trusty-dart-68765ebb-us-central1-b-2ls0\",\"flaky\":false,\"previous_flaky\":false,\"previous_result\":\"CompileTimeError\",\"previous_commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"previous_commit_time\":1551185312,\"previous_build_number\":2403,\"changed\":false}\n{\"name\":\"co19/Language/Classes/Abstract_Instance_Members/inherited_t02\",\"configuration\":\"dartk-linux-product-x64\",\"suite\":\"co19\",\"test_name\":\"Language/Classes/Abstract_Instance_Members/inherited_t02\",\"time_ms\":496,\"result\":\"CompileTimeError\",\"expected\":\"CompileTimeError\",\"matches\":true,\"commit_time\":1551185312,\"commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"build_number\":\"2404\",\"builder_name\":\"vm-kernel-linux-product-x64\",\"bot_name\":\"trusty-dart-68765ebb-us-central1-b-2ls0\",\"flaky\":false,\"previous_flaky\":false,\"previous_result\":\"CompileTimeError\",\"previous_commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"previous_commit_time\":1551185312,\"previous_build_number\":2403,\"changed\":false}\n",
+ "--logs",
+ "{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"name\": \"Flaky/Test/1\", \"configuration\": \"example_configuration\"}\n{\"name\": \"Flaky/Test/2\", \"configuration\": \"example_configuration\"}\n{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"test\":\"test1\", \"configuration\": \"example_configuration\", \"logs\": \"\u0085\"}\n{\"name\": \"Flaky/Test/1\", \"configuration\": \"example_configuration\"}\n{\"name\": \"Flaky/Test/2\", \"configuration\": \"example_configuration\"}\n",
+ "--logs-only",
+ "--flaky"
+ ],
+ "cwd": "[CACHE]/builder/sdk",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "find ignored flaky test failure logs",
+ "~followup_annotations": [
+ "@@@STEP_LOG_END@raw_io.output_text@@@"
+ ]
+ },
+ {
+ "cmd": [
"luci-auth",
"token",
"-scopes",
diff --git a/recipes/dart/neo.expected/builders_dart2js-win-debug-x64-firefox-try.json b/recipes/dart/neo.expected/builders_dart2js-win-debug-x64-firefox-try.json
index 8da3c64..6bef5f5 100644
--- a/recipes/dart/neo.expected/builders_dart2js-win-debug-x64-firefox-try.json
+++ b/recipes/dart/neo.expected/builders_dart2js-win-debug-x64-firefox-try.json
@@ -1507,6 +1507,43 @@
},
{
"cmd": [
+ "[CACHE]/builder/sdk/tools/sdks/dart-sdk/bin/dart",
+ "[CACHE]/builder/sdk/tools/bots/compare_results.dart",
+ "--flakiness-data",
+ "",
+ "--human",
+ "--verbose",
+ "[CACHE]/builder/sdk/LATEST/results.json",
+ "{\"name\":\"test_foo\",\"configuration\":\"config_bar\",\"suite\":\"suite_baz\",\"test_name\":\"Language/Classes/Abstract_Instance_Members/inherited_t01\",\"time_ms\":451,\"result\":\"CompileTimeError\",\"expected\":\"CompileTimeError\",\"matches\":true,\"commit_time\":1551185312,\"commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"build_number\":\"2404\",\"builder_name\":\"vm-kernel-linux-product-x64\",\"bot_name\":\"trusty-dart-68765ebb-us-central1-b-2ls0\",\"flaky\":false,\"previous_flaky\":false,\"previous_result\":\"CompileTimeError\",\"previous_commit_hash\":\"f0042a32250a8a6193e6d07e2b6508b13f43c864\",\"previous_commit_time\":1551185312,\"previous_build_number\":2403,\"changed\":false}\n",
+ "--logs",
+ "{\"configuration\":\"mock_configuration\"}\n{\"configuration\":\"mock_configuration\"}\n",
+ "--logs-only",
+ "--flaky"
+ ],
+ "cwd": "[CACHE]/builder/sdk",
+ "env": {
+ "ANALYZER_STATE_LOCATION_OVERRIDE": "[CLEANUP]/analysis-cache"
+ },
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "find ignored flaky test failure logs",
+ "~followup_annotations": [
+ "@@@STEP_LOG_END@raw_io.output_text@@@"
+ ]
+ },
+ {
+ "cmd": [
"luci-auth",
"token",
"-scopes",
