blob: 86806f7bf394fa3de912ca214736380b2be6b024 [file] [log] [blame]
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon
# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
# Required properties:
#
# * description: free form string
# describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
# tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
# icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
# run device tests on red Cirrus builds.
# * required_agent_capabilities: a list of strings
# list of capabilities a devicelab agent must have to be able to run this task.
#
# Optional properties:
#
# * flaky: boolean true or false
# whether the task is considered flaky; the result of running a flaky task does not affect
# the overall build status. Tests should be marked flaky when newly added, until they have
# been proved for a few cycles. Tests that are actually flaky but not being actively worked
# on should be hidden because they just cause confusion otherwise.
# * on_luci: boolean true or false
# whether the task runs on LUCI builders. Tests will be gradually marked on_luci during the
# devicelab to LUCI migration. And when the migration is complete, on_luci flag will be
# removed.
tasks:
# Tests of compiling in a variety of modes
complex_layout_ios__compile:
description: >
Collects various performance metrics of compiling the Complex
Layout sample app for iOS from Mac.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
complex_layout_win__compile:
description: >
Collects various performance metrics of compiling the Complex
Layout for Android from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
basic_material_app_ios__compile:
description: >
Collects various performance metrics of compiling the default
app for iOS from Mac.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
basic_material_app_win__compile:
description: >
Collects various performance metrics of compiling the default
app for Android from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
flutter_gallery_ios__compile:
description: >
Collects various performance metrics of compiling the Flutter
Gallery for iOS from Mac.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flutter_gallery_win__compile:
description: >
Collects various performance metrics of compiling the Flutter
Gallery for Android from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
windows_chrome_dev_mode:
description: >
Run flutter web on the devicelab and hot restart.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
# iOS on-device tests
flutter_gallery_sksl_warmup__transition_perf_e2e_ios32:
description: >
Measures the runtime performance of Flutter gallery transitions on iPhone4s
with SkSL shader warm-up with e2e.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios32"]
flaky: true
ios_defines_test:
description: >
Builds a Framework with a --dart-define and verifies it can be used as a constant
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
ios_content_validation_test:
description: >
Builds an obfuscated app and verifies contents and structure
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
on_luci: true
tiles_scroll_perf_ios__timeline_summary:
description: >
Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_views_scroll_perf_ios__timeline_summary:
description: >
Measures the runtime performance of platform views in the Complex Layout sample app on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flutter_gallery_ios32__start_up:
description: >
Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S).
stage: devicelab_ios
required_agent_capabilities: ["mac/ios32"]
flavors_test_ios:
description: >
Checks that flavored builds work on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
external_ui_integration_test_ios:
description: >
Checks that external UIs work on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
channels_integration_test_ios:
description: >
Checks that platform channels work on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_interaction_test_ios:
description: >
Checks platform interaction on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_channel_sample_test_ios:
description: >
Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_channel_sample_test_swift:
description: >
Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_view_ios__start_up:
description: >
Verifies that Platform View can be used from an iOS project.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
backdrop_filter_perf_ios__timeline_summary:
description: >
Measures the runtime performance of backdrop filter blurs on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
post_backdrop_filter_perf_ios__timeline_summary:
description: >
Measures the runtime performance of animations after a backdrop filter is removed on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
complex_layout_scroll_perf_ios__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flutter_gallery_ios__start_up:
description: >
Measures the startup time of the Flutter Gallery app on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
complex_layout_ios__start_up:
description: >
Measures the startup time of the Complex Layout sample app on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flutter_gallery_ios__transition_perf:
description: >
Measures the performance of screen transitions in Flutter Gallery on
iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
hello_world_ios__compile:
description: >
Measures the IPA size of Hello World.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
microbenchmarks_ios:
description: >
Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6.
stage: devicelab_ios
#TODO(godofredoc): Remove ios/12 capability once
# https://github.com/flutter/flutter/issues/49635 is fixed.
required_agent_capabilities: ["mac/ios", "ios/12"]
flutter_view_ios__start_up:
description: >
Verifies that Flutter View can be used from an iOS project.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
integration_ui_ios_driver:
description: >
Runs end-to-end Flutter tests on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
integration_ui_ios_keyboard_resize:
description: >
Runs end-to-end Flutter tests on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
integration_ui_ios_screenshot:
description: >
Runs end-to-end Flutter tests on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
integration_ui_ios_textfield:
description: >
Runs end-to-end Flutter tests on iPhone 6.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
ios_platform_view_tests:
description: >
Runs end-to-end tests with platform views in the scene.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
macos_chrome_dev_mode:
description: >
Run flutter web on the devicelab and hot restart.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
simple_animation_perf_ios:
description: >
Measure CPU/GPU usage percentages of a simple animation.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
smoke_catalina_hot_mode_dev_cycle_ios__benchmark:
description: >
A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
stage: devicelab_ios
required_agent_capabilities: ["mac-catalina/ios"]
smoke_catalina_start_up:
description: >
A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
stage: devicelab
required_agent_capabilities: ["mac-catalina/android"]
# macOS target platform tests
hot_mode_dev_cycle_macos_target__benchmark:
description: >
Checks the functionality and performance of hot reload on a macOS target platform
stage: devicelab
required_agent_capabilities: ["mac/ios"]
ios_app_with_extensions_test:
description: >
Checks that an iOS app with extensions can be built for physical and simulated devices.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
# Tests running on Windows host
flavors_test_win:
description: >
Checks that flavored builds work on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
channels_integration_test_win:
description: >
Checks that platform channels work when app is launched from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
hot_mode_dev_cycle_win__benchmark:
description: >
Measures the performance of Dart VM hot patching feature on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
# Tests running on Linux hosts
flutter_gallery__transition_perf_e2e_ios32:
description: >
Measures the performance of screen transitions in Flutter Gallery on
Android with e2e.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios32"]
flaky: true
flutter_gallery__transition_perf_e2e_ios:
description: >
Measures the performance of screen transitions in Flutter Gallery on
Android with e2e.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
new_gallery_ios__transition_perf:
description: >
Measures the performance of screen transitions in the new Flutter Gallery on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
large_image_changer_perf_ios:
description: >
Measures memory, cpu, and gpu usage when rotating through a series of large images.
stage: devicelab
required_agent_capabilities: ["mac/ios"]
# android_splash_screen_integration_test:
# description: >
# Runs end-to-end test of Flutter's Android splash behavior.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
# flaky: true