| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| |
| from recipe_engine.post_process import ( |
| DoesNotRun, |
| DoesNotRunRE, |
| DropExpectation, |
| Filter, |
| MustRun, |
| MustRunRE, |
| StatusException, |
| StatusFailure, |
| StatusSuccess, |
| StepCommandContains, |
| StepException, |
| StepFailure, |
| ) |
| |
| DEPS = [ |
| 'dart', |
| 'build', |
| 'depot_tools/tryserver', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/platform', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/runtime', |
| 'recipe_engine/step', |
| 'recipe_engine/swarming', |
| ] |
| |
| |
| CANNED_OUTPUT_DIR = { |
| 'logs.json': |
| '{"test":"test1", "configuration": "example_configuration"}\n', |
| 'results.json': |
| '{"name":"test1", "configuration": example_configuration"}\n' |
| '{"name":"test2", "configuration": example_configuration_2"}\n', |
| } |
| |
| CANNED_FLAKY_OUTPUT_DIR = { |
| 'logs.json': |
| '{"name": "Flaky/Test/1", "configuration": "example_configuration"}\n' |
| '{"name": "Flaky/Test/2", "configuration": "example_configuration"}\n', |
| 'results.json': |
| '{"name":"Flaky/Test/1", "configuration": "example_configuration"}\n' |
| '{"name":"Flaky/Test/2", "configuration": "example_configuration"}\n', |
| } |
| |
| |
| TRIGGER_RESULT = { |
| "results": [ |
| { |
| "build": { |
| "id": "8963024236039183808", |
| "tags": [ |
| "builder:analyzer-linux-release", |
| "parent_buildername:dart-sdk-linux" |
| ], |
| "url": "https://ci.chromium.org/p/dart/builds/b8963024236039183808", |
| } |
| }, |
| { |
| "build": { |
| "id": "8963024236039836208", |
| "tags": [ |
| "builder:analyzer-strong-linux-release", |
| "parent_buildername:dart-sdk-linux" |
| ], |
| "url": "https://ci.chromium.org/p/dart/builds/b8963024236039836208", |
| } |
| }, |
| { |
| "build": { |
| "id": "8963024236039228096", |
| "tags": [ |
| "builder:analyzer-analysis-server-linux", |
| "parent_buildername:dart-sdk-linux" |
| ], |
| "url": "https://ci.chromium.org/p/dart/builds/b8963024236039228096", |
| } |
| } |
| ] |
| } |
| |
| |
| TEST_MATRIX = { |
| "filesets": { |
| "test": "[]", |
| "trigger": "[]" |
| }, |
| "global": { |
| "chrome": "66.0.3359.139", |
| "firefox": "61" |
| }, |
| "builder_configurations": [{ |
| "builders": ["vm-kernel-win-release-simarm_x64",], |
| "meta": {}, |
| "steps": [{ |
| "name": |
| "gn", |
| "script": |
| "tools/gn.py", |
| "arguments": [ |
| "--bytecode", "--gen-snapshot=${build_root}/gen_snapshot" |
| ] |
| }, { |
| "name": "build", |
| "script": "tools/build.py", |
| "arguments": ["--arch=x64", "runtime"] |
| }, { |
| "name": "test1", |
| "script": "tools/test.py", |
| "arguments": [ |
| "foo", "-ndartk-${system}-${mode}-${arch}", "language_2", |
| "co19_2/sel", "--exclude_suite=co19" |
| ], |
| "shards": 2, |
| "fileset": "test" |
| }, { |
| "name": "test2", |
| "arguments": ["foo", "--bar", "-ndartk-${system}-${mode}-${arch}"], |
| }, { |
| "name": "trigger", |
| "fileset": "trigger", |
| "trigger": ["foo-builder", "bar-builder"] |
| }, { |
| "name": "dart", |
| "script": "out/ReleaseX64/dart", |
| "arguments": ["--bar", "foo.dart"] |
| }, { |
| "name": "test3", |
| "arguments": ["-ndartk-${system}-${mode}-${arch}", "foo", "--bar"], |
| "fileset": "test", |
| "shards": 2 |
| }] |
| }, { |
| "builders": ["co19",], |
| "meta": {}, |
| "steps": [{ |
| "name": "co19", |
| "arguments": ["co19/nnbd", "co19_2"], |
| }] |
| }, { |
| "builders": ["co19_sharded",], |
| "meta": {}, |
| "steps": [{ |
| "name": "co19", |
| "arguments": ["co19/nnbd", "co19_2"], |
| "fileset": "test", |
| "shards": 1 |
| }] |
| }, { |
| "builders": ["analyzer-linux-release",], |
| "meta": {}, |
| "steps": [{ |
| "name": "build", |
| "script": "tools/build.py" |
| }, { |
| "name": "test1", |
| "script": "tools/test.py", |
| "fileset": "test", |
| "shards": 2, |
| "arguments": ["-nunittest-asserts-${mode}-${system}"] |
| }, { |
| "name": "test2", |
| "arguments": ["foo", "--bar", "-ndartk-${system}-${mode}-${arch}"], |
| }, { |
| "name": "trigger", |
| "fileset": "trigger", |
| "trigger": ["foo-builder", "bar-builder"] |
| }, { |
| "name": "test3", |
| "arguments": ["-nanalyzer-asserts-${system}", "foo", "--bar"], |
| "fileset": "test", |
| "shards": 2 |
| }] |
| }, { |
| "builders": [ |
| "dart2js-strong-mac-x64-chrome", "dart2js-strong-linux-x64-firefox", |
| "dart2js-strong-win-x64-chrome" |
| ], |
| "meta": {}, |
| "steps": [{ |
| "name": "test1", |
| "script": "tools/test.py", |
| "arguments": [ |
| "-ndart2js-${system}-${runtime}", "foo", "--bar", |
| "-e co19, language_2" |
| ], |
| "shards": 2, |
| "fileset": "test" |
| }, { |
| "name": "custom", |
| "script": "tools/custom_thing.py", |
| "arguments": ["foo", "--bar", "--buildername"] |
| }, { |
| "name": "custom_runner", |
| "script": "tools/custom_test_runner.py", |
| "testRunner": True, |
| "arguments": ["foo", "--bar", "--buildername"] |
| }, { |
| "name": |
| "test2", |
| "arguments": [ |
| "-ndart2js-${system}-${runtime}", "foo", "--bar", "co19" |
| ], |
| }] |
| }, { |
| "builders": ["vm-kernel-mac-release-x64"], |
| "meta": {}, |
| "steps": [{ |
| "name": "build", |
| "script": "tools/build.py", |
| "arguments": [] |
| }, { |
| "name": "custom", |
| "script": "out/custom_thing", |
| "arguments": ["foo", "--bar", "--buildername"] |
| }] |
| }, { |
| "builders": ["vm-kernel-precomp-android-release-arm64"], |
| "meta": {}, |
| "steps": [{ |
| "name": "android", |
| "shards": 2, |
| "fileset": "test", |
| "arguments": ["-ndartkp-android-${mode}-${arch}"] |
| }] |
| }, { |
| "builders": ["fuzz-linux"], |
| "steps": [{ |
| "name": "make a fuzz", |
| "script": "out/ReleaseX64/dart", |
| "arguments": [ |
| "runtime/tools/dartfuzz/dartfuzz_test.dart", "--isolates", "8", |
| "--no-show-stats", "--time", "2700" |
| ], |
| "shards": 3, |
| "fileset": "test" |
| }] |
| }] |
| } |
| |
| RESULT_DATA = ( |
| '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t01"' |
| ',"configuration":"dartk-linux-product-x64","suite":"co19_2",' |
| '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t01",' |
| '"time_ms":451,"result":"CompileTimeError","expected":"CompileTimeError",' |
| '"matches":true,"commit_time":1551185312,' |
| '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",' |
| '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,' |
| '"previous_flaky":false,"previous_result":"CompileTimeError",' |
| '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"previous_commit_time":1551185312,"previous_build_number":2403,' |
| '"changed":false}\n' + |
| '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t02"' |
| ',"configuration":"dartk-linux-product-x64","suite":"co19_2",' |
| '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t02",' |
| '"time_ms":496,"result":"CompileTimeError","expected":"CompileTimeError",' |
| '"matches":true,"commit_time":1551185312,' |
| '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",' |
| '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,' |
| '"previous_flaky":false,"previous_result":"CompileTimeError",' |
| '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"previous_commit_time":1551185312,"previous_build_number":2403,' |
| '"changed":false}\n') |
| |
| RESULT_DATA_MULTIPLE_CONFIGURATIONS = ( |
| '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t01"' |
| ',"configuration":"dartk-linux-product-x64","suite":"co19_2",' |
| '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t01",' |
| '"time_ms":451,"result":"CompileTimeError","expected":"CompileTimeError",' |
| '"matches":true,"commit_time":1551185312,' |
| '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",' |
| '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,' |
| '"previous_flaky":false,"previous_result":"CompileTimeError",' |
| '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"previous_commit_time":1551185312,"previous_build_number":2403,' |
| '"changed":false}\n' + |
| '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t02"' |
| ',"configuration":"dartk-linux-release-x64","suite":"co19_2",' |
| '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t02",' |
| '"time_ms":496,"result":"CompileTimeError","expected":"CompileTimeError",' |
| '"matches":true,"commit_time":1551185312,' |
| '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",' |
| '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,' |
| '"previous_flaky":false,"previous_result":"CompileTimeError",' |
| '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",' |
| '"previous_commit_time":1551185312,"previous_build_number":2403,' |
| '"changed":false}\n') |
| |
| |
| def RunSteps(api): |
| api.dart.checkout( |
| api.properties.get('clobber', False), |
| api.properties.get('custom_vars', {})) |
| |
| api.dart.kill_tasks() |
| api.dart.read_debug_log() |
| |
| api.dart.test(test_data=TEST_MATRIX) |
| |
| |
| def _canned_step(api, name, shards=0, local_shard=True, deflake=True): |
| step_name = '%s_shard_%s' % (name, 1) if shards > 0 else name |
| data = None |
| for i in range(1 if shards > 0 else 0, shards + 1): |
| if not data: |
| data = _canned_output_dir(api, step_name) |
| else: |
| data += _canned_output_dir(api, '%s_shard_%s' % (name, i)) |
| if deflake: |
| deflaking_name = name if local_shard else '%s_shard_1' % name |
| data += _canned_output_dir( |
| api, deflaking_name, CANNED_FLAKY_OUTPUT_DIR, prefix='deflaking.') |
| if deflake: |
| data += api.step_data('deflaking.list tests to deflake (%s)' % name, |
| stdout=api.raw_io.output('Flaky/Test/1\nFlaky/Test/2')) |
| |
| return data |
| |
| |
| def _canned_output_dir(api, step_name, content=None, prefix=''): |
| content = content or CANNED_OUTPUT_DIR |
| data = api.empty_test_data() |
| for filename in ('logs.json', 'results.json'): |
| data += api.step_data('%sread %s for %s' % (prefix, filename, step_name), |
| api.file.read_text(text_content=content[filename])) |
| return data |
| |
| |
| def _bot_script(name): |
| return '[CACHE]/builder/sdk/tools/bots/%s' % name |
| |
| |
| def _upload_isolate_step_data(api, fileset='test'): |
| return api.step_data( |
| 'upload testing fileset %s' % fileset, |
| stdout=api.raw_io.output('uploaded digest: %s_hash/47' % fileset)) |
| |
| |
| def GenTests(api): |
| UPDATE_BLAMELIST_STEP_RE = ".*blamelist.*" |
| UPDATE_BLAMELIST_STEP = ('update blamelists for configuration ' |
| '"dartk-linux-product-x64"') |
| |
| yield api.test( |
| 'basic', |
| api.properties( |
| shard_timeout='600', |
| custom_vars={'download_firefox': True}, |
| python_version='python3'), |
| api.buildbucket.try_build( |
| build_number=1357, |
| builder='dart2js-strong-linux-x64-firefox-try', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'test1', 2, False), |
| _canned_step(api, 'test2'), |
| _canned_step(api, 'custom_runner', deflake=False), |
| _upload_isolate_step_data(api), |
| api.tryserver.gerrit_change_target_ref('refs/heads/stable'), |
| api.step_data('download previous results.gsutil find latest build', |
| api.raw_io.output_text('123', name='latest')), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(DoesNotRunRE, UPDATE_BLAMELIST_STEP_RE), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'branch-builder-does-not-publish-results', |
| api.properties(shard_timeout='600'), |
| api.buildbucket.ci_build( |
| build_number=1357, |
| builder='co19', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart', |
| git_ref="refs/heads/some-branch"), |
| _canned_step(api, 'co19'), |
| api.step_data('download previous results.gsutil find latest build', |
| api.raw_io.output_text('123', name='latest')), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(MustRunRE, '.*.gsutil upload results.json'), |
| api.post_process(DoesNotRunRE, '.*publish results to pub/sub'), |
| api.post_process(StepCommandContains, 'test results', |
| [_bot_script('compare_results.dart')]), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'ci-test-data-branch-builder-does-publish-results', |
| api.properties(shard_timeout='600'), |
| api.buildbucket.ci_build( |
| build_number=1357, |
| builder='co19', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart', |
| git_ref="refs/heads/ci-test-data"), |
| _canned_step(api, 'co19'), |
| api.step_data('download previous results.gsutil find latest build', |
| api.raw_io.output_text('123', name='latest')), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(MustRunRE, '.*.gsutil upload results.json'), |
| api.post_process(MustRunRE, '.*publish results to pub/sub'), |
| api.post_process(StepCommandContains, 'test results', |
| [_bot_script('get_builder_status.dart')]), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'release-builder-does-not-publish-results', |
| api.properties(shard_timeout='600'), |
| api.buildbucket.ci_build( |
| build_number=1357, |
| builder='co19-dev', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart', |
| git_ref="refs/heads/dev"), |
| _canned_step(api, 'co19'), |
| api.step_data('download previous results.gsutil find latest build', |
| api.raw_io.output_text('123', name='latest')), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(MustRunRE, '.*.gsutil upload results.json'), |
| api.post_process(DoesNotRunRE, '.*publish results to pub/sub'), |
| api.post_process(StepCommandContains, 'test results', |
| [_bot_script('compare_results.dart')]), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'bisection-does-not-refer-to-latest-build-or-publish-results', |
| api.properties( |
| shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711), |
| api.buildbucket.ci_build( |
| build_number=1357, |
| builder='dart2js-strong-linux-x64-firefox', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'test1', 2, False), |
| _canned_step(api, 'test2'), |
| _canned_step(api, 'custom_runner', deflake=False), |
| _upload_isolate_step_data(api), |
| api.step_data( |
| 'add fields to result records', |
| api.raw_io.output_text(RESULT_DATA_MULTIPLE_CONFIGURATIONS)), |
| api.post_process(DoesNotRunRE, '.*find latest build'), |
| api.post_process(DoesNotRunRE, '.*publish results to pub/sub'), |
| api.post_process(StepCommandContains, 'test results', |
| [_bot_script('compare_results.dart')]), |
| api.post_process(MustRun, UPDATE_BLAMELIST_STEP), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'bisection-on-test-branch-updates-blamelist', |
| api.properties( |
| shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711), |
| api.buildbucket.ci_build( |
| build_number=1357, |
| builder='dart2js-strong-linux-x64-firefox', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart', |
| git_ref="refs/heads/ci-test-data"), |
| _canned_step(api, 'test1', 2, False), |
| _canned_step(api, 'test2'), |
| _canned_step(api, 'custom_runner', deflake=False), |
| _upload_isolate_step_data(api), |
| api.step_data( |
| 'add fields to result records', |
| api.raw_io.output_text(RESULT_DATA_MULTIPLE_CONFIGURATIONS)), |
| api.post_process(DoesNotRunRE, '.*find latest build'), |
| api.post_process(DoesNotRunRE, '.*publish results to pub/sub'), |
| api.post_process(StepCommandContains, 'test results', |
| [_bot_script('compare_results.dart')]), |
| # Verify that the test branch builder updates the staging database |
| api.post_process(StepCommandContains, UPDATE_BLAMELIST_STEP, ['-s']), |
| api.post_process(StatusSuccess), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'experimental-bisection-build-does-not-update-blamelist', |
| api.properties( |
| shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711), |
| api.buildbucket.ci_build( |
| build_number=1357, |
| builder='dart2js-strong-linux-x64-firefox', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.runtime(is_experimental=True), |
| _canned_step(api, 'test1', 2, False), |
| _canned_step(api, 'test2'), |
| _canned_step(api, 'custom_runner', deflake=False), |
| _upload_isolate_step_data(api), |
| api.step_data( |
| 'add fields to result records', |
| api.raw_io.output_text(RESULT_DATA_MULTIPLE_CONFIGURATIONS)), |
| api.post_process(DoesNotRunRE, '.*find latest build'), |
| api.post_process(DoesNotRunRE, '.*publish results to pub/sub'), |
| api.post_process(StepCommandContains, 'test results', |
| [_bot_script('compare_results.dart')]), |
| api.post_process(DoesNotRunRE, UPDATE_BLAMELIST_STEP_RE), |
| api.post_process(StatusSuccess), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'analyzer-linux-release', |
| api.properties(bot_id='trusty-dart-123'), |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='analyzer-linux-release', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'test1', 2, False), |
| _canned_step(api, 'test2'), |
| _canned_step(api, 'test3', 2), |
| _upload_isolate_step_data(api, 'test'), |
| _upload_isolate_step_data(api, 'trigger'), |
| api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'experimental-builds-are-side-effect-free', |
| api.properties(bot_id='trusty-dart-123'), |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='analyzer-linux-release', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'test1', 2, False), |
| _canned_step(api, 'test2'), |
| _canned_step(api, 'test3', 2), |
| api.runtime(is_experimental=True), |
| _upload_isolate_step_data(api, 'test'), |
| _upload_isolate_step_data(api, 'trigger'), |
| api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(DoesNotRunRE, r'.*publish results.*'), |
| api.post_process(StatusSuccess), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'build-failure-in-matrix', |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='analyzer-linux-release', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data('build', retcode=1), |
| api.post_process(DoesNotRun, 'test1'), |
| api.post_process(StepFailure, 'build'), |
| api.post_process(StatusFailure), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'basic-missing-name', |
| api.buildbucket.ci_build( |
| builder='this-name-does-not-exist-in-test-matrix', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.post_process(StatusException), |
| ) |
| |
| |
| yield api.test( |
| 'failed-tests', |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='co19_sharded', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'co19', 1, False), |
| _upload_isolate_step_data(api), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.step_data('test results', retcode=1), |
| api.post_process(StepFailure, 'test results'), |
| api.post_process(StatusFailure), |
| api.post_process(Filter('test results')), |
| ) |
| |
| yield api.test( |
| 'failed-to-get-test-results', |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='co19_sharded', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'co19', 1, False), |
| _upload_isolate_step_data(api), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.step_data('test results', retcode=2), |
| api.post_process(StepException, 'test results'), |
| api.post_process(StatusException), |
| api.post_process(Filter('test results')), |
| ) |
| |
| yield api.test( |
| 'vm-win', |
| api.platform('win', 64), |
| api.properties(clobber=True, bot_id='win-dart-123'), |
| api.path.exists(api.path['checkout'].join('out')), |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='vm-kernel-win-release-simarm_x64', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _upload_isolate_step_data(api, 'test'), |
| _upload_isolate_step_data(api, 'trigger'), |
| api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'basic-mac', |
| api.platform('mac', 64), |
| api.properties(bot_id='mac-dart-123'), |
| api.buildbucket.ci_build( |
| revision='a' * 40, |
| build_number=1357, |
| builder='dart2js-strong-mac-x64-chrome-dev', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.properties(clobber=True, custom_vars={'download_chrome': True}), |
| api.path.exists(api.path['checkout'].join('xcodebuild')), |
| _upload_isolate_step_data(api), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'example-mac', |
| api.platform('mac', 64), |
| api.buildbucket.ci_build( |
| builder='vm-kernel-mac-release-x64', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'example-android', |
| api.platform('linux', 64), |
| api.buildbucket.ci_build( |
| builder='vm-kernel-precomp-android-release-arm64', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'android', 2, False), |
| _upload_isolate_step_data(api), |
| api.post_process(StatusSuccess), |
| ) |
| |
| yield api.test( |
| 'non-test-step-error-is-step-failure', |
| api.buildbucket.ci_build( |
| builder='vm-kernel-win-release-simarm_x64', |
| git_repo='https://dart.googlesource.com/sdk'), |
| _upload_isolate_step_data(api, 'test'), |
| _upload_isolate_step_data(api, 'trigger'), |
| api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)), |
| api.step_data('dart', retcode=1), |
| api.post_process(StepFailure, 'dart'), |
| api.post_process(Filter('dart')), |
| ) |
| |
| chunk_upload_step = ('upload new results.' |
| 'upload test results to big query.' |
| 'upload results chunk to big query') |
| yield api.test( |
| 'upload-error-is-infra-failure', |
| api.buildbucket.ci_build( |
| builder='co19', git_repo='https://dart.googlesource.com/sdk'), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.step_data(chunk_upload_step, retcode=1), |
| api.post_process(StepException, chunk_upload_step), |
| api.post_process(StatusException), |
| api.post_process(Filter(chunk_upload_step)), |
| ) |
| |
| download_step = 'download previous results.gsutil download previous results' |
| yield api.test( |
| 'download-results-error-is-infra-failure', |
| api.buildbucket.ci_build( |
| builder='co19', git_repo='https://dart.googlesource.com/sdk'), |
| api.step_data('download previous results.gsutil find latest build', |
| api.raw_io.output_text("1234", name="latest")), |
| api.step_data(download_step, retcode=1), |
| api.post_process(StepException, download_step), |
| api.post_process(StatusException), |
| api.post_process(Filter(download_step)), |
| ) |
| |
| deflake_step = 'deflaking.list tests to deflake (co19)' |
| yield api.test( |
| 'failed-to-get-tests-to-deflake-is-infra-failure', |
| api.buildbucket.ci_build( |
| builder='co19_sharded', git_repo='https://dart.googlesource.com/sdk'), |
| _upload_isolate_step_data(api), |
| api.step_data(deflake_step, retcode=1), |
| api.post_process(StepException, deflake_step), |
| api.post_process(StatusException), |
| api.post_process(Filter(deflake_step)), |
| ) |
| |
| update_flaky_step = 'update flakiness information' |
| yield api.test( |
| 'failed-to-update-flaky-data-is-infra-failure', |
| api.buildbucket.ci_build( |
| builder='co19_sharded', git_repo='https://dart.googlesource.com/sdk'), |
| _upload_isolate_step_data(api), |
| api.step_data(update_flaky_step, retcode=1), |
| api.post_process(StepException, update_flaky_step), |
| api.post_process(StatusException), |
| api.post_process(Filter(update_flaky_step)), |
| ) |
| |
| update_latest_step = 'upload new results.gsutil update "latest" reference' |
| yield api.test( |
| 'failed-to-update-latest-is-infra-failure', |
| api.buildbucket.ci_build( |
| builder='co19_sharded', git_repo='https://dart.googlesource.com/sdk'), |
| _upload_isolate_step_data(api), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.step_data(update_latest_step, retcode=1), |
| api.post_process(StepException, update_latest_step), |
| api.post_process(StatusException), |
| api.post_process(Filter(update_latest_step)), |
| ) |
| |
| upload_step = 'upload new results.gsutil upload revision' |
| yield api.test( |
| 'failed-to-upload-is-infra-failure', |
| api.buildbucket.ci_build( |
| builder='co19', git_repo='https://dart.googlesource.com/sdk'), |
| api.step_data(upload_step, retcode=1), |
| api.post_process(StepException, upload_step), |
| api.post_process(StatusException), |
| api.post_process(Filter(upload_step)), |
| ) |
| |
| publish_step = 'upload new results.publish results to pub/sub' |
| yield api.test( |
| 'failed-to-publish-is-infra-failure', |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='co19', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.step_data(publish_step, retcode=1), |
| api.post_process(StepException, publish_step), |
| api.post_process(StatusException), |
| api.post_process(Filter(publish_step)), |
| ) |
| |
| yield api.test( |
| 'fuzz-test', |
| _upload_isolate_step_data(api), |
| api.buildbucket.ci_build( |
| builder='fuzz-linux', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data( |
| 'make a fuzz_shard_1', |
| api.swarming.collect([ |
| api.swarming.task_result( |
| id='0', name='make a fuzz_shard_1', failure=True) |
| ])), |
| api.post_process(MustRun, 'make a fuzz_shard_2'), |
| api.post_process(StepFailure, 'make a fuzz_shard_1'), |
| api.post_process(StatusFailure), |
| ) |
| |
| yield api.test( |
| 'fuzz-shard-timeout', |
| _upload_isolate_step_data(api), |
| api.buildbucket.ci_build( |
| builder='fuzz-linux', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data( |
| 'make a fuzz_shard_1', |
| api.swarming.collect([ |
| api.swarming.task_result( |
| id='0', |
| name='make a fuzz_shard_1', |
| state=api.swarming.TaskState.TIMED_OUT) |
| ])), |
| api.post_process(MustRun, 'make a fuzz_shard_2'), |
| api.post_process(StepException, 'make a fuzz_shard_1'), |
| api.post_process(StatusException), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'test-shard-failure', |
| api.properties(bot_id='trusty-dart-123'), |
| api.buildbucket.ci_build( |
| builder='analyzer-linux-release', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data( |
| 'test1_shard_1', |
| api.swarming.collect([ |
| api.swarming.task_result( |
| id='0', name='test1_shard_1', failure=True) |
| ])), |
| _upload_isolate_step_data(api, 'test'), |
| _upload_isolate_step_data(api, 'trigger'), |
| api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)), |
| api.post_process(StepException, 'test1_shard_1'), |
| api.post_process(StatusException), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'local-test-shard-failure', |
| api.properties(bot_id='trusty-dart-123'), |
| api.buildbucket.ci_build( |
| builder='analyzer-linux-release', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data('test3_shard_2', retcode=1), |
| _upload_isolate_step_data(api, 'test'), |
| _upload_isolate_step_data(api, 'trigger'), |
| api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)), |
| api.post_process(StepException, 'test3_shard_2'), |
| api.post_process(StatusException), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'co19', |
| api.properties(bot_id='trusty-dart-123'), |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='co19_sharded', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| _canned_step(api, 'co19', 1, False), |
| _upload_isolate_step_data(api), |
| api.step_data('add fields to result records', |
| api.raw_io.output_text(RESULT_DATA)), |
| api.post_process(StatusSuccess), |
| api.post_process(Filter().include_re(r'.*co19.*')), |
| ) |
| |
| yield api.test( |
| 'co19-no-revinfo', |
| api.properties(bot_id='trusty-dart-123'), |
| api.buildbucket.ci_build( |
| revision='3456abce78ef', |
| build_number=1357, |
| builder='co19', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data('gclient get package versions', |
| api.json.output(name='revinfo', data={})), |
| api.post_process(StatusException), |
| api.post_process(Filter().include_re(r'.*(get package versions|co19).*')), |
| ) |
| |
| yield api.test( |
| 'custom-test-runner-failure-is-infra-failure', |
| api.properties(shard_timeout='600'), |
| api.buildbucket.try_build( |
| build_number=1357, |
| builder='dart2js-strong-linux-x64-firefox-try', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data('custom_runner', retcode=1), |
| _upload_isolate_step_data(api, 'test'), |
| api.post_process(StepException, 'custom_runner'), |
| api.post_process(StatusException), |
| api.post_process(DropExpectation), |
| ) |
| |
| yield api.test( |
| 'invalid-fileset-upload-output', |
| api.buildbucket.ci_build( |
| builder='co19_sharded', |
| git_repo='https://dart.googlesource.com/sdk', |
| project='dart'), |
| api.step_data( |
| 'upload testing fileset test', stdout=api.raw_io.output('')), |
| api.post_process(StatusException), |
| api.post_process(Filter('upload testing fileset test')), |
| ) |
