Dynamically add integration_tests and screenshots to native iOS test results (#95704)
diff --git a/packages/integration_test/example/integration_test/_extended_test_io.dart b/packages/integration_test/example/integration_test/_extended_test_io.dart
index 377aa42..8c2456a 100644
--- a/packages/integration_test/example/integration_test/_extended_test_io.dart
+++ b/packages/integration_test/example/integration_test/_extended_test_io.dart
@@ -25,6 +25,24 @@
// Build our app.
app.main();
+ // Pump a frame.
+ await tester.pumpAndSettle();
+
+ // Verify that platform version is retrieved.
+ expect(
+ find.byWidgetPredicate(
+ (Widget widget) =>
+ widget is Text &&
+ widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
+ ),
+ findsOneWidget,
+ );
+ });
+
+ testWidgets('verify screenshot', (WidgetTester tester) async {
+ // Build our app.
+ app.main();
+
// On Android, this is required prior to taking the screenshot.
await binding.convertFlutterSurfaceToImage();
@@ -39,15 +57,5 @@
expect(secondPng.isNotEmpty, isTrue);
expect(listEquals(firstPng, secondPng), isTrue);
-
- // Verify that platform version is retrieved.
- expect(
- find.byWidgetPredicate(
- (Widget widget) =>
- widget is Text &&
- widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
- ),
- findsOneWidget,
- );
});
}
diff --git a/packages/integration_test/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/packages/integration_test/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
index 1e6676d..bfcb53e 100644
--- a/packages/integration_test/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
+++ b/packages/integration_test/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
@@ -20,6 +20,20 @@
ReferencedContainer = "container:Runner.xcodeproj">
</BuildableReference>
</BuildActionEntry>
+ <BuildActionEntry
+ buildForTesting = "YES"
+ buildForRunning = "NO"
+ buildForProfiling = "NO"
+ buildForArchiving = "NO"
+ buildForAnalyzing = "YES">
+ <BuildableReference
+ BuildableIdentifier = "primary"
+ BlueprintIdentifier = "769541C723A0351900E5C350"
+ BuildableName = "RunnerTests.xctest"
+ BlueprintName = "RunnerTests"
+ ReferencedContainer = "container:Runner.xcodeproj">
+ </BuildableReference>
+ </BuildActionEntry>
</BuildActionEntries>
</BuildAction>
<TestAction
diff --git a/packages/integration_test/example/ios/RunnerTests/RunnerTests.m b/packages/integration_test/example/ios/RunnerTests/RunnerTests.m
index edd7f10..d3bdcc6 100644
--- a/packages/integration_test/example/ios/RunnerTests/RunnerTests.m
+++ b/packages/integration_test/example/ios/RunnerTests/RunnerTests.m
@@ -5,4 +5,89 @@
@import XCTest;
@import integration_test;
+#pragma mark - Dynamic tests
+
INTEGRATION_TEST_IOS_RUNNER(RunnerTests)
+
+@interface RunnerTests (DynamicTests)
+@end
+
+@implementation RunnerTests (DynamicTests)
+
+- (void)setUp {
+ // Verify tests have been dynamically added from FLUTTER_TARGET=integration_test/extended_test.dart
+ XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyScreenshot")]);
+ XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyText")]);
+ XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"screenshotPlaceholder")]);
+}
+
+@end
+
+#pragma mark - Fake test results
+
+@interface IntegrationTestPlugin ()
+- (instancetype)initForRegistration;
+@end
+
+@interface FLTIntegrationTestRunner ()
+@property IntegrationTestPlugin *integrationTestPlugin;
+@end
+
+@interface FakeIntegrationTestPlugin : IntegrationTestPlugin
+@property(nonatomic, nullable) NSDictionary<NSString *, NSString *> *testResults;
+@end
+
+@implementation FakeIntegrationTestPlugin
+@synthesize testResults;
+
+- (void)setupChannels:(id<FlutterBinaryMessenger>)binaryMessenger {
+}
+
+@end
+
+#pragma mark - Behavior tests
+
+@interface IntegrationTestTests : XCTestCase
+@end
+
+@implementation IntegrationTestTests
+
+- (void)testDeprecatedIntegrationTest {
+ NSString *testResult;
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+ BOOL testPass = [[IntegrationTestIosTest new] testIntegrationTest:&testResult];
+#pragma clang diagnostic pop
+ XCTAssertTrue(testPass, @"%@", testResult);
+}
+
+- (void)testMethodNamesFromDartTests {
+ XCTAssertEqualObjects([FLTIntegrationTestRunner
+ testCaseNameFromDartTestName:@"this is a test"], @"testThisIsATest");
+ XCTAssertEqualObjects([FLTIntegrationTestRunner
+ testCaseNameFromDartTestName:@"VALIDATE multi-point 🚀 UNICODE123: 😁"], @"testValidateMultiPointUnicode123");
+ XCTAssertEqualObjects([FLTIntegrationTestRunner
+ testCaseNameFromDartTestName:@"!UPPERCASE:\\ lower_seperate?"], @"testUppercaseLowerSeperate");
+}
+
+- (void)testDuplicatedDartTests {
+ FakeIntegrationTestPlugin *fakePlugin = [[FakeIntegrationTestPlugin alloc] initForRegistration];
+ // These are unique test names in dart, but would result in duplicate
+ // XCTestCase names when the emojis are stripped.
+ fakePlugin.testResults = @{@"unique": @"dart test failure", @"emoji 🐢": @"success", @"emoji 🐇": @"failure"};
+
+ FLTIntegrationTestRunner *runner = [[FLTIntegrationTestRunner alloc] init];
+ runner.integrationTestPlugin = fakePlugin;
+
+ NSMutableDictionary<NSString *, NSString *> *failuresByTestName = [[NSMutableDictionary alloc] init];
+ [runner testIntegrationTestWithResults:^(SEL nativeTestSelector, BOOL success, NSString *failureMessage) {
+ NSString *testName = NSStringFromSelector(nativeTestSelector);
+ XCTAssertFalse([failuresByTestName.allKeys containsObject:testName]);
+ failuresByTestName[testName] = failureMessage;
+ }];
+ XCTAssertEqualObjects(failuresByTestName,
+ (@{@"testUnique": @"dart test failure",
+ @"testDuplicateTestNames": @"Cannot test \"emoji 🐇\", duplicate XCTestCase tests named testEmoji"}));
+}
+
+@end
diff --git a/packages/integration_test/ios/Classes/FLTIntegrationTestRunner.h b/packages/integration_test/ios/Classes/FLTIntegrationTestRunner.h
new file mode 100644
index 0000000..65568fc
--- /dev/null
+++ b/packages/integration_test/ios/Classes/FLTIntegrationTestRunner.h
@@ -0,0 +1,36 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@import Foundation;
+
+@class UIImage;
+
+NS_ASSUME_NONNULL_BEGIN
+
+typedef void (^FLTIntegrationTestResults)(SEL nativeTestSelector, BOOL success, NSString *_Nullable failureMessage);
+
+@interface FLTIntegrationTestRunner : NSObject
+
+/**
+ * Any screenshots captured by the plugin.
+ */
+@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
+
+/**
+ * Starts dart tests and waits for results.
+ *
+ * @param testResult Will be called once per every completed dart test.
+ */
+- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult;
+
+/**
+ * An appropriate XCTest method name based on the dart test name.
+ *
+ * Example: dart test "verify widget-ABC123" becomes "testVerifyWidgetABC123"
+ */
++ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName;
+
+@end
+
+NS_ASSUME_NONNULL_END
diff --git a/packages/integration_test/ios/Classes/FLTIntegrationTestRunner.m b/packages/integration_test/ios/Classes/FLTIntegrationTestRunner.m
new file mode 100644
index 0000000..286465c
--- /dev/null
+++ b/packages/integration_test/ios/Classes/FLTIntegrationTestRunner.m
@@ -0,0 +1,77 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "FLTIntegrationTestRunner.h"
+
+#import "IntegrationTestPlugin.h"
+
+@import ObjectiveC.runtime;
+@import UIKit;
+
+@interface FLTIntegrationTestRunner ()
+
+@property IntegrationTestPlugin *integrationTestPlugin;
+
+@end
+
+@implementation FLTIntegrationTestRunner
+
+- (instancetype)init {
+ self = [super init];
+ _integrationTestPlugin = [IntegrationTestPlugin instance];
+
+ return self;
+}
+
+- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult {
+ IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
+ UIViewController *rootViewController = UIApplication.sharedApplication.delegate.window.rootViewController;
+ if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
+ testResult(NSSelectorFromString(@"testSetup"), NO, @"rootViewController was not expected FlutterViewController");
+ }
+ FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
+ [integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
+
+ // Spin the runloop.
+ while (!integrationTestPlugin.testResults) {
+ [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]];
+ }
+
+ NSMutableSet<NSString *> *testCaseNames = [[NSMutableSet alloc] init];
+
+ [integrationTestPlugin.testResults enumerateKeysAndObjectsUsingBlock:^(NSString *test, NSString *result, BOOL *stop) {
+ NSString *testSelectorName = [[self class] testCaseNameFromDartTestName:test];
+
+ // Validate Objective-C test names are unique after sanitization.
+ if ([testCaseNames containsObject:testSelectorName]) {
+ NSString *reason = [NSString stringWithFormat:@"Cannot test \"%@\", duplicate XCTestCase tests named %@", test, testSelectorName];
+ testResult(NSSelectorFromString(@"testDuplicateTestNames"), NO, reason);
+ *stop = YES;
+ return;
+ }
+ [testCaseNames addObject:testSelectorName];
+ SEL testSelector = NSSelectorFromString(testSelectorName);
+
+ if ([result isEqualToString:@"success"]) {
+ testResult(testSelector, YES, nil);
+ } else {
+ testResult(testSelector, NO, result);
+ }
+ }];
+}
+
+- (NSDictionary<NSString *,UIImage *> *)capturedScreenshotsByName {
+ return self.integrationTestPlugin.capturedScreenshotsByName;
+}
+
++ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName {
+ NSString *capitalizedString = dartTestName.localizedCapitalizedString;
+ // Objective-C method names must be alphanumeric.
+ NSCharacterSet *disallowedCharacters = NSCharacterSet.alphanumericCharacterSet.invertedSet;
+ // Remove disallowed characters.
+ NSString *upperCamelTestName = [[capitalizedString componentsSeparatedByCharactersInSet:disallowedCharacters] componentsJoinedByString:@""];
+ return [NSString stringWithFormat:@"test%@", upperCamelTestName];
+}
+
+@end
diff --git a/packages/integration_test/ios/Classes/IntegrationTestIosTest.h b/packages/integration_test/ios/Classes/IntegrationTestIosTest.h
index cac3be4..33aaea5 100644
--- a/packages/integration_test/ios/Classes/IntegrationTestIosTest.h
+++ b/packages/integration_test/ios/Classes/IntegrationTestIosTest.h
@@ -2,16 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#import <Foundation/Foundation.h>
+@import Foundation;
+@import ObjectiveC.runtime;
NS_ASSUME_NONNULL_BEGIN
-@protocol FLTIntegrationTestScreenshotDelegate;
-
+DEPRECATED_MSG_ATTRIBUTE("Use FLTIntegrationTestRunner instead.")
@interface IntegrationTestIosTest : NSObject
-- (instancetype)initWithScreenshotDelegate:(nullable id<FLTIntegrationTestScreenshotDelegate>)delegate NS_DESIGNATED_INITIALIZER;
-
/**
* Initiate dart tests and wait for results. @c testResult will be set to a string describing the results.
*
@@ -21,26 +19,48 @@
@end
+// For every Flutter dart test, dynamically generate an Objective-C method mirroring the test results
+// so it is reported as a native XCTest run result.
+// If the Flutter dart tests have captured screenshots, add them to the XCTest bundle.
#define INTEGRATION_TEST_IOS_RUNNER(__test_class) \
- @interface __test_class : XCTestCase<FLTIntegrationTestScreenshotDelegate> \
+ @interface __test_class : XCTestCase \
@end \
\
@implementation __test_class \
\
- - (void)testIntegrationTest { \
- NSString *testResult; \
- IntegrationTestIosTest *integrationTestIosTest = integrationTestIosTest = [[IntegrationTestIosTest alloc] initWithScreenshotDelegate:self]; \
- BOOL testPass = [integrationTestIosTest testIntegrationTest:&testResult]; \
- XCTAssertTrue(testPass, @"%@", testResult); \
- } \
- \
- - (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(NSString *)name { \
- XCTAttachment *attachment = [XCTAttachment attachmentWithImage:screenshot]; \
- attachment.lifetime = XCTAttachmentLifetimeKeepAlways; \
- if (name != nil) { \
- attachment.name = name; \
+ + (NSArray<NSInvocation *> *)testInvocations { \
+ FLTIntegrationTestRunner *integrationTestRunner = [[FLTIntegrationTestRunner alloc] init]; \
+ NSMutableArray<NSInvocation *> *testInvocations = [[NSMutableArray alloc] init]; \
+ [integrationTestRunner testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *failureMessage) { \
+ IMP assertImplementation = imp_implementationWithBlock(^(id _self) { \
+ XCTAssertTrue(success, @"%@", failureMessage); \
+ }); \
+ class_addMethod(self, testSelector, assertImplementation, "v@:"); \
+ NSMethodSignature *signature = [self instanceMethodSignatureForSelector:testSelector]; \
+ NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; \
+ invocation.selector = testSelector; \
+ [testInvocations addObject:invocation]; \
+ }]; \
+ NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName = integrationTestRunner.capturedScreenshotsByName; \
+ if (capturedScreenshotsByName.count > 0) { \
+ IMP screenshotImplementation = imp_implementationWithBlock(^(id _self) { \
+ [capturedScreenshotsByName enumerateKeysAndObjectsUsingBlock:^(NSString *name, UIImage *screenshot, BOOL *stop) { \
+ XCTAttachment *attachment = [XCTAttachment attachmentWithImage:screenshot]; \
+ attachment.lifetime = XCTAttachmentLifetimeKeepAlways; \
+ if (name != nil) { \
+ attachment.name = name; \
+ } \
+ [_self addAttachment:attachment]; \
+ }]; \
+ }); \
+ SEL attachmentSelector = NSSelectorFromString(@"screenshotPlaceholder"); \
+ class_addMethod(self, attachmentSelector, screenshotImplementation, "v@:"); \
+ NSMethodSignature *attachmentSignature = [self instanceMethodSignatureForSelector:attachmentSelector]; \
+ NSInvocation *attachmentInvocation = [NSInvocation invocationWithMethodSignature:attachmentSignature]; \
+ attachmentInvocation.selector = attachmentSelector; \
+ [testInvocations addObject:attachmentInvocation]; \
} \
- [self addAttachment:attachment]; \
+ return testInvocations; \
} \
\
@end
diff --git a/packages/integration_test/ios/Classes/IntegrationTestIosTest.m b/packages/integration_test/ios/Classes/IntegrationTestIosTest.m
index 6a54ed2..808fa94 100644
--- a/packages/integration_test/ios/Classes/IntegrationTestIosTest.m
+++ b/packages/integration_test/ios/Classes/IntegrationTestIosTest.m
@@ -3,61 +3,40 @@
// found in the LICENSE file.
#import "IntegrationTestIosTest.h"
-#import "IntegrationTestPlugin.h"
-@interface IntegrationTestIosTest()
-@property (nonatomic) IntegrationTestPlugin *integrationTestPlugin;
-@end
+#import "IntegrationTestPlugin.h"
+#import "FLTIntegrationTestRunner.h"
+
+#pragma mark - Deprecated
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-implementations"
@implementation IntegrationTestIosTest
-- (instancetype)initWithScreenshotDelegate:(id<FLTIntegrationTestScreenshotDelegate>)delegate {
- self = [super init];
- _integrationTestPlugin = [IntegrationTestPlugin instance];
- _integrationTestPlugin.screenshotDelegate = delegate;
- return self;
-}
-
-- (instancetype)init {
- return [self initWithScreenshotDelegate:nil];
-}
-
- (BOOL)testIntegrationTest:(NSString **)testResult {
- IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
-
- UIViewController *rootViewController =
- [[[[UIApplication sharedApplication] delegate] window] rootViewController];
- if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
- NSLog(@"expected FlutterViewController as rootViewController.");
- return NO;
- }
- FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
- [integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
- while (!integrationTestPlugin.testResults) {
- CFRunLoopRunInMode(kCFRunLoopDefaultMode, 1.f, NO);
- }
- NSDictionary<NSString *, NSString *> *testResults = integrationTestPlugin.testResults;
- NSMutableArray<NSString *> *passedTests = [NSMutableArray array];
- NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
NSLog(@"==================== Test Results =====================");
- for (NSString *test in testResults.allKeys) {
- NSString *result = testResults[test];
- if ([result isEqualToString:@"success"]) {
- NSLog(@"%@ passed.", test);
- [passedTests addObject:test];
+ NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
+ NSMutableArray<NSString *> *testNames = [NSMutableArray array];
+ [[FLTIntegrationTestRunner new] testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *message) {
+ NSString *testName = NSStringFromSelector(testSelector);
+ [testNames addObject:testName];
+ if (success) {
+ NSLog(@"%@ passed.", testName);
} else {
- NSLog(@"%@ failed: %@", test, result);
- [failedTests addObject:test];
+ NSLog(@"%@ failed: %@", testName, message);
+ [failedTests addObject:testName];
}
- }
+ }];
NSLog(@"================== Test Results End ====================");
BOOL testPass = failedTests.count == 0;
- if (!testPass && testResult) {
+ if (!testPass && testResult != NULL) {
*testResult =
[NSString stringWithFormat:@"Detected failed integration test(s) %@ among %@",
- failedTests.description, testResults.allKeys.description];
+ failedTests.description, testNames.description];
}
return testPass;
}
@end
+#pragma clang diagnostic pop
diff --git a/packages/integration_test/ios/Classes/IntegrationTestPlugin.h b/packages/integration_test/ios/Classes/IntegrationTestPlugin.h
index 9684835..4836339 100644
--- a/packages/integration_test/ios/Classes/IntegrationTestPlugin.h
+++ b/packages/integration_test/ios/Classes/IntegrationTestPlugin.h
@@ -6,13 +6,6 @@
NS_ASSUME_NONNULL_BEGIN
-@protocol FLTIntegrationTestScreenshotDelegate
-
-/** This will be called when a dart integration test triggers a window screenshot with @c takeScreenshot. */
-- (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(nullable NSString *)name;
-
-@end
-
/** A Flutter plugin that's responsible for communicating the test results back
* to iOS XCTest. */
@interface IntegrationTestPlugin : NSObject <FlutterPlugin>
@@ -23,6 +16,11 @@
*/
@property(nonatomic, readonly, nullable) NSDictionary<NSString *, NSString *> *testResults;
+/**
+ * Mapping of screenshot images by suggested names, captured by the dart tests.
+ */
+@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
+
/** Fetches the singleton instance of the plugin. */
+ (IntegrationTestPlugin *)instance;
@@ -30,8 +28,6 @@
- (instancetype)init NS_UNAVAILABLE;
-@property(weak, nonatomic) id<FLTIntegrationTestScreenshotDelegate> screenshotDelegate;
-
@end
NS_ASSUME_NONNULL_END
diff --git a/packages/integration_test/ios/Classes/IntegrationTestPlugin.m b/packages/integration_test/ios/Classes/IntegrationTestPlugin.m
index 82d2635..a8a80b6 100644
--- a/packages/integration_test/ios/Classes/IntegrationTestPlugin.m
+++ b/packages/integration_test/ios/Classes/IntegrationTestPlugin.m
@@ -2,10 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-@import UIKit;
-
#import "IntegrationTestPlugin.h"
+@import UIKit;
+
static NSString *const kIntegrationTestPluginChannel = @"plugins.flutter.io/integration_test";
static NSString *const kMethodTestFinished = @"allTestsFinished";
static NSString *const kMethodScreenshot = @"captureScreenshot";
@@ -16,10 +16,13 @@
@property(nonatomic, readwrite) NSDictionary<NSString *, NSString *> *testResults;
+- (instancetype)init NS_DESIGNATED_INITIALIZER;
+
@end
@implementation IntegrationTestPlugin {
NSDictionary<NSString *, NSString *> *_testResults;
+ NSMutableDictionary<NSString *, UIImage *> *_capturedScreenshotsByName;
}
+ (IntegrationTestPlugin *)instance {
@@ -32,7 +35,13 @@
}
- (instancetype)initForRegistration {
- return [super init];
+ return [self init];
+}
+
+- (instancetype)init {
+ self = [super init];
+ _capturedScreenshotsByName = [NSMutableDictionary new];
+ return self;
}
+ (void)registerWithRegistrar:(NSObject<FlutterPluginRegistrar> *)registrar {
@@ -59,7 +68,7 @@
// If running as a native Xcode test, attach to test.
UIImage *screenshot = [self capturePngScreenshot];
NSString *name = call.arguments[@"name"];
- [self.screenshotDelegate didTakeScreenshot:screenshot attachmentName:name];
+ _capturedScreenshotsByName[name] = screenshot;
// Also pass back along the channel for the driver to handle.
NSData *pngData = UIImagePNGRepresentation(screenshot);
