Rework the integration test timeout logic (#9322)
diff --git a/packages/devtools_app/integration_test/test/live_connection/app_test.dart b/packages/devtools_app/integration_test/test/live_connection/app_test.dart index 6603595..2b40eaa 100644 --- a/packages/devtools_app/integration_test/test/live_connection/app_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/app_test.dart
@@ -25,7 +25,9 @@ await resetHistory(); }); - testWidgets('connect to app and switch tabs', (tester) async { + testWidgets('connect to app and switch tabs', timeout: shortTimeout, ( + tester, + ) async { await pumpAndConnectDevTools(tester, testApp); // For the sake of this test, do not show extension screens by default.
diff --git a/packages/devtools_app/integration_test/test/live_connection/debugger_panel_test.dart b/packages/devtools_app/integration_test/test/live_connection/debugger_panel_test.dart index feb7865..58b5a09 100644 --- a/packages/devtools_app/integration_test/test/live_connection/debugger_panel_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/debugger_panel_test.dart
@@ -29,7 +29,7 @@ expect(testApp.vmServiceUri, isNotNull); }); - testWidgets('Debugger panel', (tester) async { + testWidgets('Debugger panel', timeout: longTimeout, (tester) async { await pumpAndConnectDevTools(tester, testApp); await switchToScreen( tester,
diff --git a/packages/devtools_app/integration_test/test/live_connection/devtools_extensions_test.dart b/packages/devtools_app/integration_test/test/live_connection/devtools_extensions_test.dart index 5dfb145..377b03d 100644 --- a/packages/devtools_app/integration_test/test/live_connection/devtools_extensions_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/devtools_extensions_test.dart
@@ -41,7 +41,9 @@ resetDevToolsExtensionEnabledStates(); }); - testWidgets('end to end extensions flow', (tester) async { + testWidgets('end to end extensions flow', timeout: mediumTimeout, ( + tester, + ) async { await pumpDevTools(tester); // TODO(https://github.com/flutter/devtools/issues/9196): re-enable this
diff --git a/packages/devtools_app/integration_test/test/live_connection/eval_and_browse_test.dart b/packages/devtools_app/integration_test/test/live_connection/eval_and_browse_test.dart index f37ba18..3fa80f7 100644 --- a/packages/devtools_app/integration_test/test/live_connection/eval_and_browse_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/eval_and_browse_test.dart
@@ -30,7 +30,7 @@ await resetHistory(); }); - testWidgets('memory eval and browse', (tester) async { + testWidgets('memory eval and browse', timeout: mediumTimeout, (tester) async { await pumpAndConnectDevTools(tester, testApp); final evalTester = EvalTester(tester);
diff --git a/packages/devtools_app/integration_test/test/live_connection/eval_and_inspect_test.dart b/packages/devtools_app/integration_test/test/live_connection/eval_and_inspect_test.dart index 43baef7..be03ece 100644 --- a/packages/devtools_app/integration_test/test/live_connection/eval_and_inspect_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/eval_and_inspect_test.dart
@@ -32,7 +32,9 @@ await resetHistory(); }); - testWidgets('eval with scope in inspector window', (tester) async { + testWidgets('eval with scope in inspector window', timeout: mediumTimeout, ( + tester, + ) async { await pumpAndConnectDevTools(tester, testApp); final evalTester = EvalTester(tester); @@ -47,6 +49,7 @@ testWidgets( 'eval with scope on widget tree node', + timeout: mediumTimeout, (tester) async { await pumpAndConnectDevTools(tester, testApp);
diff --git a/packages/devtools_app/integration_test/test/live_connection/export_snapshot_test.dart b/packages/devtools_app/integration_test/test/live_connection/export_snapshot_test.dart index e5d5779..673c3c5 100644 --- a/packages/devtools_app/integration_test/test/live_connection/export_snapshot_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/export_snapshot_test.dart
@@ -25,7 +25,7 @@ await resetHistory(); }); - testWidgets('Export snapshot', (tester) async { + testWidgets('Export snapshot', timeout: shortTimeout, (tester) async { await pumpAndConnectDevTools(tester, testApp); await prepareMemoryUI(tester); await takeHeapSnapshot(tester);
diff --git a/packages/devtools_app/integration_test/test/live_connection/performance_screen_event_recording_test.dart b/packages/devtools_app/integration_test/test/live_connection/performance_screen_event_recording_test.dart index 2a00a8d..dd34a1c 100644 --- a/packages/devtools_app/integration_test/test/live_connection/performance_screen_event_recording_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/performance_screen_event_recording_test.dart
@@ -24,7 +24,9 @@ expect(testApp.vmServiceUri, isNotNull); }); - testWidgets('can process and refresh timeline data', (tester) async { + testWidgets('can process and refresh timeline data', timeout: longTimeout, ( + tester, + ) async { await pumpAndConnectDevTools(tester, testApp); logStatus(
diff --git a/packages/devtools_app/integration_test/test/live_connection/service_connection_test.dart b/packages/devtools_app/integration_test/test/live_connection/service_connection_test.dart index c672a11..93e3f5f 100644 --- a/packages/devtools_app/integration_test/test/live_connection/service_connection_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/service_connection_test.dart
@@ -27,7 +27,9 @@ await resetHistory(); }); - testWidgets('initial service connection state', (tester) async { + testWidgets('initial service connection state', timeout: mediumTimeout, ( + tester, + ) async { await pumpAndConnectDevTools(tester, testApp); // Await a delay to ensure the service extensions have had a chance to @@ -87,17 +89,16 @@ ); logStatus('verify managers have all been initialized'); - expect(serviceConnection.serviceManager.isolateManager, isNotNull); expect(serviceConnection.serviceManager.serviceExtensionManager, isNotNull); - expect(serviceConnection.vmFlagManager, isNotNull); expect( serviceConnection.serviceManager.isolateManager.isolates.value, isNotEmpty, ); expect(serviceConnection.vmFlagManager.flags.value, isNotNull); - if (serviceConnection.serviceManager.isolateManager.selectedIsolate.value == - null) { + final selectedIsolate = + serviceConnection.serviceManager.isolateManager.selectedIsolate; + if (selectedIsolate.value == null) { await whenValueNonNull( serviceConnection.serviceManager.isolateManager.selectedIsolate, );
diff --git a/packages/devtools_app/integration_test/test/live_connection/service_extensions_test.dart b/packages/devtools_app/integration_test/test/live_connection/service_extensions_test.dart index 8b011a0..18105d3 100644 --- a/packages/devtools_app/integration_test/test/live_connection/service_extensions_test.dart +++ b/packages/devtools_app/integration_test/test/live_connection/service_extensions_test.dart
@@ -37,7 +37,7 @@ testWidgets( 'can call services and service extensions', - + timeout: mediumTimeout, (tester) async { await pumpAndConnectDevTools(tester, testApp); await tester.pump(longDuration); @@ -75,58 +75,62 @@ skip: true, // https://github.com/flutter/devtools/issues/8107 ); - testWidgets('loads initial extension states from device', (tester) async { - await pumpAndConnectDevTools(tester, testApp); - await tester.pump(longDuration); + testWidgets( + 'loads initial extension states from device', + timeout: const Timeout(Duration(minutes: 3)), + (tester) async { + await pumpAndConnectDevTools(tester, testApp); + await tester.pump(longDuration); - // Ensure all futures are completed before running checks. - final service = serviceConnection.serviceManager.service!; - await service.allFuturesCompleted; + // Ensure all futures are completed before running checks. + final service = serviceConnection.serviceManager.service!; + await service.allFuturesCompleted; - final serviceExtensionsToEnable = [ - (extensions.debugPaint.extension, true), - (extensions.slowAnimations.extension, 5.0), - (extensions.togglePlatformMode.extension, 'iOS'), - ]; + final serviceExtensionsToEnable = [ + (extensions.debugPaint.extension, true), + (extensions.slowAnimations.extension, 5.0), + (extensions.togglePlatformMode.extension, 'iOS'), + ]; - logStatus('enabling service extensions on the test device'); - // Enable a service extension of each type (boolean, numeric, string). - for (final ext in serviceExtensionsToEnable) { - await serviceConnection.serviceManager.serviceExtensionManager - .setServiceExtensionState(ext.$1, enabled: true, value: ext.$2); - } + logStatus('enabling service extensions on the test device'); + // Enable a service extension of each type (boolean, numeric, string). + for (final ext in serviceExtensionsToEnable) { + await serviceConnection.serviceManager.serviceExtensionManager + .setServiceExtensionState(ext.$1, enabled: true, value: ext.$2); + } - logStatus('disconnecting from the test device'); - await disconnectFromTestApp(tester); + logStatus('disconnecting from the test device'); + await disconnectFromTestApp(tester); - for (final ext in serviceExtensionsToEnable) { - expect( - serviceConnection.serviceManager.serviceExtensionManager - .isServiceExtensionAvailable(ext.$1), - isFalse, - ); - } + for (final ext in serviceExtensionsToEnable) { + expect( + serviceConnection.serviceManager.serviceExtensionManager + .isServiceExtensionAvailable(ext.$1), + isFalse, + ); + } - logStatus('reconnecting to the test device'); - await connectToTestApp(tester, testApp); + logStatus('reconnecting to the test device'); + await connectToTestApp(tester, testApp); - logStatus('verify extension states have been restored from the device'); - for (final ext in serviceExtensionsToEnable) { - expect( - serviceConnection.serviceManager.serviceExtensionManager - .isServiceExtensionAvailable(ext.$1), - isTrue, - reason: 'Expect ${ext.$1} to be available', - ); - await _verifyExtensionStateInServiceManager( - ext.$1, - enabled: true, - value: ext.$2, - ); - } + logStatus('verify extension states have been restored from the device'); + for (final ext in serviceExtensionsToEnable) { + expect( + serviceConnection.serviceManager.serviceExtensionManager + .isServiceExtensionAvailable(ext.$1), + isTrue, + reason: 'Expect ${ext.$1} to be available', + ); + await _verifyExtensionStateInServiceManager( + ext.$1, + enabled: true, + value: ext.$2, + ); + } - await disconnectFromTestApp(tester); - }); + await disconnectFromTestApp(tester); + }, + ); } Future<void> _verifyBooleanExtension(WidgetTester tester) async {
diff --git a/packages/devtools_app/integration_test/test/offline/memory_offline_data_test.dart b/packages/devtools_app/integration_test/test/offline/memory_offline_data_test.dart index 15461c2..86a4969 100644 --- a/packages/devtools_app/integration_test/test/offline/memory_offline_data_test.dart +++ b/packages/devtools_app/integration_test/test/offline/memory_offline_data_test.dart
@@ -14,7 +14,9 @@ void main() { IntegrationTestWidgetsFlutterBinding.ensureInitialized(); - testWidgets('Memory screen can load offline data', (tester) async { + testWidgets('Memory screen can load offline data', timeout: mediumTimeout, ( + tester, + ) async { await pumpDevTools(tester); logStatus('1 - pumped devtools'); await loadSampleData(tester, memoryFileName);
diff --git a/packages/devtools_app/integration_test/test/offline/perfetto_test.dart b/packages/devtools_app/integration_test/test/offline/perfetto_test.dart index 1cefd06..f2fb3c3 100644 --- a/packages/devtools_app/integration_test/test/offline/perfetto_test.dart +++ b/packages/devtools_app/integration_test/test/offline/perfetto_test.dart
@@ -19,6 +19,7 @@ testWidgets( 'Perfetto trace viewer loads data and scrolls for Flutter frames', + timeout: mediumTimeout, (tester) async { await pumpDevTools(tester); await loadSampleData(tester, performanceFileName);
diff --git a/packages/devtools_shared/lib/src/test/integration_test_runner.dart b/packages/devtools_shared/lib/src/test/integration_test_runner.dart index 6c7dd67..ed39069 100644 --- a/packages/devtools_shared/lib/src/test/integration_test_runner.dart +++ b/packages/devtools_shared/lib/src/test/integration_test_runner.dart
@@ -115,15 +115,13 @@ ); bool testTimedOut = false; - final timeout = Future.delayed(const Duration(minutes: 8)).then((_) { + await process.exitCode.timeout(const Duration(minutes: 8), onTimeout: () { testTimedOut = true; + // TODO(srawlins): Refactor the retry situation to catch a + // TimeoutException, and not recursively call `runTest`. + return -1; }); - await Future.any([ - process.exitCode, - timeout, - ]); - debugLog( 'shutting down processes because ' '${testTimedOut ? 'test timed out' : 'test finished'}',
diff --git a/packages/devtools_test/lib/src/integration_test/integration_test_utils.dart b/packages/devtools_test/lib/src/integration_test/integration_test_utils.dart index 9074fb6..4b3d520 100644 --- a/packages/devtools_test/lib/src/integration_test/integration_test_utils.dart +++ b/packages/devtools_test/lib/src/integration_test/integration_test_utils.dart
@@ -157,3 +157,21 @@ 'last_screenshot': lastScreenshot, }); } + +/// A timeout for a "short" integration test. +/// +/// Adjust as needed; this is used to override the 10-minute or infinite timeout +/// in [testWidgets]. +const Timeout shortTimeout = Timeout(Duration(minutes: 2)); + +/// A timeout for a "medium" integration test. +/// +/// Adjust as needed; this is used to override the 10-minute or infinite timeout +/// in [testWidgets]. +const Timeout mediumTimeout = Timeout(Duration(minutes: 3)); + +/// A timeout for a "long" integration test. +/// +/// Adjust as needed; this is used to override the 10-minute or infinite timeout +/// in [testWidgets]. +const Timeout longTimeout = Timeout(Duration(minutes: 4));