Allow batching of hits
diff --git a/lib/src/usage_impl.dart b/lib/src/usage_impl.dart index 73acc24..83ef9a6 100644 --- a/lib/src/usage_impl.dart +++ b/lib/src/usage_impl.dart
@@ -57,6 +57,9 @@ static const String _defaultAnalyticsUrl = 'https://www.google-analytics.com/collect'; + static const String _defaultAnalyticsBatchingUrl = + 'https://www.google-analytics.com/batch'; + @override final String trackingId; @override @@ -76,16 +79,25 @@ AnalyticsOpt analyticsOpt = AnalyticsOpt.optOut; late final String _url; + late final String _batchingUrl; final StreamController<Map<String, dynamic>> _sendController = StreamController.broadcast(sync: true); - AnalyticsImpl(this.trackingId, this.properties, this.postHandler, - {this.applicationName, this.applicationVersion, String? analyticsUrl}) { + AnalyticsImpl( + this.trackingId, + this.properties, + this.postHandler, { + this.applicationName, + this.applicationVersion, + String? analyticsUrl, + String? analyticsBatchingUrl, + }) { if (applicationName != null) setSessionValue('an', applicationName); if (applicationVersion != null) setSessionValue('av', applicationVersion); _url = analyticsUrl ?? _defaultAnalyticsUrl; + _batchingUrl = analyticsBatchingUrl ?? _defaultAnalyticsBatchingUrl; } bool? _firstRun; @@ -118,38 +130,41 @@ @override Future sendScreenView(String viewName, {Map<String, String>? parameters}) { - var args = <String, dynamic>{'cd': viewName}; - if (parameters != null) { - args.addAll(parameters); - } - return _sendPayload('screenview', args); + var args = <String, String>{'cd': viewName, ...?parameters}; + return _enqueuePayload('screenview', args); } @override Future sendEvent(String category, String action, {String? label, int? value, Map<String, String>? parameters}) { - var args = <String, dynamic>{'ec': category, 'ea': action}; - if (label != null) args['el'] = label; - if (value != null) args['ev'] = value; - if (parameters != null) { - args.addAll(parameters); - } - return _sendPayload('event', args); + final args = <String, String>{ + 'ec': category, + 'ea': action, + if (label != null) 'el': label, + if (value != null) 'ev': value.toString(), + ...?parameters + }; + + return _enqueuePayload('event', args); } @override Future sendSocial(String network, String action, String target) { - var args = <String, dynamic>{'sn': network, 'sa': action, 'st': target}; - return _sendPayload('social', args); + var args = <String, String>{'sn': network, 'sa': action, 'st': target}; + return _enqueuePayload('social', args); } @override Future sendTiming(String variableName, int time, {String? category, String? label}) { - var args = <String, dynamic>{'utv': variableName, 'utt': time}; - if (label != null) args['utl'] = label; - if (category != null) args['utc'] = category; - return _sendPayload('timing', args); + var args = <String, String>{ + 'utv': variableName, + 'utt': time.toString(), + if (label != null) 'utl': label, + if (category != null) 'utc': category, + }; + + return _enqueuePayload('timing', args); } @override @@ -177,9 +192,29 @@ description = description.substring(0, maxExceptionLength); } - var args = <String, dynamic>{'exd': description}; - if (fatal != null && fatal) args['exf'] = '1'; - return _sendPayload('exception', args); + var args = <String, String>{ + 'exd': description, + if (fatal != null && fatal) 'exf': '1', + }; + return _enqueuePayload('exception', args); + } + + static const _batchingKey = #_batching; + + @override + Future<T> withBatching<T>( + FutureOr<T> Function() callback, { + int maxEventsPerBatch = 20, + }) async { + final queue = _BatchingQueue(maxEventsPerBatch: maxEventsPerBatch); + return await runZoned(() async { + final result = await callback(); + if (queue.enqueuedEvents.isNotEmpty) { + // Send any remaining events. + _trySendBatch(queue.takeBatch()); + } + return result; + }, zoneValues: {_batchingKey: queue}); } @override @@ -219,36 +254,66 @@ /// /// Valid values for [hitType] are: 'pageview', 'screenview', 'event', /// 'transaction', 'item', 'social', 'exception', and 'timing'. - Future sendRaw(String hitType, Map<String, dynamic> args) { - return _sendPayload(hitType, args); + Future sendRaw(String hitType, Map<String, String> args) { + return _enqueuePayload(hitType, args); } + /// Puts a single event in the queue. If batching is not enabled it will be + /// send immediately - otherwise when the batch is full (20 events) or when + /// the batching callback is over. /// Valid values for [hitType] are: 'pageview', 'screenview', 'event', /// 'transaction', 'item', 'social', 'exception', and 'timing'. - Future _sendPayload(String hitType, Map<String, dynamic> args) { - if (!enabled) return Future.value(); + Future<void> _enqueuePayload(String hitType, Map<String, String> args) async { + if (!enabled) return; - if (_bucket.removeDrop()) { - _variableMap.forEach((key, value) { - args[key] = value; - }); + final eventArgs = <String, String>{ + ...args, + ..._variableMap, + 'v': '1', // protocol version + 'tid': trackingId, + 'cid': clientId, + 't': hitType, + }; - args['v'] = '1'; // protocol version - args['tid'] = trackingId; - args['cid'] = clientId; - args['t'] = hitType; + final batch = <Map<String, String>>[]; - _sendController.add(args); - - return _recordFuture(postHandler.sendPost(_url, args)); + // See if we currently are batching events: + final batchingQueue = Zone.current[_batchingKey]; + if (batchingQueue is _BatchingQueue) { + // Add the current event to the batch. + batchingQueue.enqueuedEvents.add(eventArgs); + if (!batchingQueue.isFull) { + // Queue not full yet. Do nothing. + return; + } + // We have a full batch, start a new one. + batch.addAll(batchingQueue.takeBatch()); } else { - return Future.value(); + batch.add(eventArgs); + } + _trySendBatch(batch); + } + + void _trySendBatch(List<Map<String, String>> batch) { + if (_bucket.removeDrop()) { + for (final args in batch) { + _sendController.add(args); + } + + // See if we currently are batching events: + final batchingQueue = Zone.current[_batchingKey]; + if (batchingQueue is _BatchingQueue) { + _recordFuture(postHandler.sendPost(_batchingUrl, batch)); + } else { + assert(batch.length == 1); + _recordFuture(postHandler.sendPost(_url, batch)); + } } } - Future _recordFuture(Future f) { + void _recordFuture(Future f) { _futures.add(f); - return f.whenComplete(() => _futures.remove(f)); + f.whenComplete(() => _futures.remove(f)); } } @@ -283,8 +348,24 @@ /// The `Future` from [sendPost] should complete when the operation is finished, /// but failures to send the information should be silent. abstract class PostHandler { - Future sendPost(String url, Map<String, dynamic> parameters); + Future sendPost(String url, List<Map<String, dynamic>> batch); /// Free any used resources. void close(); } + +class _BatchingQueue { + List<Map<String, String>> enqueuedEvents = []; + int maxEventsPerBatch; + + _BatchingQueue({required this.maxEventsPerBatch}); + + bool get isFull => enqueuedEvents.length >= maxEventsPerBatch; + List<Map<String, String>> takeBatch() { + final result = enqueuedEvents.take(maxEventsPerBatch).toList(); + // If for some reason more events than one batch have queued up simply + // discard them. + enqueuedEvents.clear(); + return result; + } +}
diff --git a/lib/src/usage_impl_html.dart b/lib/src/usage_impl_html.dart index 1ebe299..9ee8632 100644 --- a/lib/src/usage_impl_html.dart +++ b/lib/src/usage_impl_html.dart
@@ -39,13 +39,14 @@ HtmlPostHandler({this.mockRequestor}); @override - Future sendPost(String url, Map<String, dynamic> parameters) { + Future sendPost(String url, List<Map<String, dynamic>> batch) { var viewportWidth = document.documentElement!.clientWidth; var viewportHeight = document.documentElement!.clientHeight; - parameters['vp'] = '${viewportWidth}x$viewportHeight'; - - var data = postEncode(parameters); + var data = batch + .map((event) => + postEncode({...event, 'vp': '${viewportWidth}x$viewportHeight'})) + .join('\n'); Future<HttpRequest> Function(String, {String method, dynamic sendData}) requestor = mockRequestor ?? HttpRequest.request; return requestor(url, method: 'POST', sendData: data).catchError((e) {
diff --git a/lib/src/usage_impl_io.dart b/lib/src/usage_impl_io.dart index 62388ad..c696275 100644 --- a/lib/src/usage_impl_io.dart +++ b/lib/src/usage_impl_io.dart
@@ -3,10 +3,11 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; -import 'dart:convert' show jsonDecode, JsonEncoder; +import 'dart:convert' show JsonEncoder, jsonDecode; import 'dart:io'; import 'package:path/path.dart' as path; +import 'package:meta/meta.dart'; import 'usage_impl.dart'; @@ -24,12 +25,12 @@ class AnalyticsIO extends AnalyticsImpl { AnalyticsIO( String trackingId, String applicationName, String applicationVersion, - {String? analyticsUrl, Directory? documentDirectory}) + {String? analyticsUrl, Directory? documentDirectory, HttpClient? client}) : super( trackingId, IOPersistentProperties(applicationName, documentDirPath: documentDirectory?.path), - IOPostHandler(), + IOPostHandler(client: client), applicationName: applicationName, applicationVersion: applicationVersion, analyticsUrl: analyticsUrl) { @@ -40,7 +41,8 @@ } } -String _createUserAgent() { +@visibleForTesting +String createUserAgent() { final locale = getPlatformLocale() ?? ''; if (Platform.isAndroid) { @@ -74,24 +76,16 @@ } class IOPostHandler extends PostHandler { - final String _userAgent; - final HttpClient? mockClient; + final HttpClient _client; - HttpClient? _client; - - IOPostHandler({this.mockClient}) : _userAgent = _createUserAgent(); + IOPostHandler({HttpClient? client}) + : _client = (client ?? HttpClient())..userAgent = createUserAgent(); @override - Future sendPost(String url, Map<String, dynamic> parameters) async { - var data = postEncode(parameters); - - if (_client == null) { - _client = mockClient ?? HttpClient(); - _client!.userAgent = _userAgent; - } - + Future sendPost(String url, List<Map<String, dynamic>> batch) async { + var data = batch.map(postEncode).join('\n'); try { - var req = await _client!.postUrl(Uri.parse(url)); + var req = await _client.postUrl(Uri.parse(url)); req.write(data); var response = await req.close(); await response.drain(); @@ -102,7 +96,7 @@ } @override - void close() => _client?.close(); + void close() => _client.close(); } JsonEncoder _jsonEncoder = JsonEncoder.withIndent(' ');
diff --git a/lib/usage.dart b/lib/usage.dart index cd548fe..d30b105 100644 --- a/lib/usage.dart +++ b/lib/usage.dart
@@ -98,6 +98,15 @@ Future sendTiming(String variableName, int time, {String? category, String? label}); + /// All sendX calls sent from within [callback] will be enqueued and sent + /// in batches of {maxEventsPerBatch} messages at a time. + /// + /// Google analytics by default supports no more than 20 items per batch. + Future<T> withBatching<T>( + FutureOr<T> Function() callback, { + int maxEventsPerBatch = 20, + }); + /// Start a timer. The time won't be calculated, and the analytics information /// sent, until the [AnalyticsTimer.finish] method is called. AnalyticsTimer startTimer(String variableName, @@ -283,6 +292,12 @@ return Future.value(); } + + @override + Future<T> withBatching<T>(FutureOr<T> Function() callback) { + // TODO: implement withBatching + throw UnimplementedError(); + } } /// Sanitize a stacktrace. This will shorten file paths in order to remove any
diff --git a/pubspec.yaml b/pubspec.yaml index 9613c34..c26017a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml
@@ -14,5 +14,6 @@ path: ^1.8.0-nullsafety dev_dependencies: + mockito: ^5.0.0-nullsafety.7 pedantic: ^1.9.0 test: ^1.16.0-nullsafety
diff --git a/test/src/common.dart b/test/src/common.dart index 304faf9..ccebb72 100644 --- a/test/src/common.dart +++ b/test/src/common.dart
@@ -50,8 +50,8 @@ List<Map<String, dynamic>> sentValues = []; @override - Future sendPost(String url, Map<String, dynamic> parameters) { - sentValues.add(parameters); + Future sendPost(String url, List<Map<String, dynamic>> batch) { + sentValues.addAll(batch); return Future.value(); }
diff --git a/test/usage_impl_io_test.dart b/test/usage_impl_io_test.dart index 8b15606..c0a97f0 100644 --- a/test/usage_impl_io_test.dart +++ b/test/usage_impl_io_test.dart
@@ -16,11 +16,17 @@ void defineTests() { group('IOPostHandler', () { test('sendPost', () async { - var httpClient = MockHttpClient(); - var postHandler = IOPostHandler(mockClient: httpClient); - var args = <String, dynamic>{'utv': 'varName', 'utt': 123}; + var mockClient = MockHttpClient(); + + var postHandler = IOPostHandler(client: mockClient); + var args = [ + <String, String>{'utv': 'varName', 'utt': '123'}, + ]; await postHandler.sendPost('http://www.google.com', args); - expect(httpClient.sendCount, 1); + expect(mockClient.requests.single.buffer.toString(), ''' +Request to http://www.google.com with ${createUserAgent()} +utv=varName&utt=123'''); + expect(mockClient.requests.single.response.drained, isTrue); }); }); @@ -49,55 +55,104 @@ expect(getPlatformLocale(), isNotNull); }); }); + + group('batching', () { + test('Without batching sends to regular url', () async { + final mockClient = MockHttpClient(); + + final analytics = AnalyticsIO( + '<TRACKING-ID', + 'usage-test', + '0.0.1', + client: mockClient, + ); + await analytics.sendEvent('my-event', 'something'); + expect(mockClient.requests.single.buffer.toString(), ''' +Request to https://www.google-analytics.com/collect with ${createUserAgent()} +ec=my-event&ea=something&an=usage-test&av=0.0.1&ul=en-us&v=1&tid=%3CTRACKING-ID&cid=8e3fa343-70bc-4afe-ad81-5fed4256b4e8&t=event'''); + }); + + test('with batching sends to batching url', () async { + var mockClient = MockHttpClient(); + + final analytics = AnalyticsIO('<TRACKING-ID', 'usage-test', '0.0.1', + client: mockClient); + await analytics.withBatching(() async { + await analytics.sendEvent('my-event1', 'something1'); + await analytics.sendEvent('my-event2', 'something2'); + await analytics.sendEvent('my-event3', 'something3'); + await analytics.sendEvent('my-event4', 'something4'); + }, maxEventsPerBatch: 3); + await analytics.sendEvent('my-event-not-batched', 'something'); + + expect(mockClient.requests.length, 3); + expect(mockClient.requests[0].buffer.toString(), ''' +Request to https://www.google-analytics.com/batch with ${createUserAgent()} +ec=my-event1&ea=something1&an=usage-test&av=0.0.1&ul=en-us&v=1&tid=%3CTRACKING-ID&cid=8e3fa343-70bc-4afe-ad81-5fed4256b4e8&t=event +ec=my-event2&ea=something2&an=usage-test&av=0.0.1&ul=en-us&v=1&tid=%3CTRACKING-ID&cid=8e3fa343-70bc-4afe-ad81-5fed4256b4e8&t=event +ec=my-event3&ea=something3&an=usage-test&av=0.0.1&ul=en-us&v=1&tid=%3CTRACKING-ID&cid=8e3fa343-70bc-4afe-ad81-5fed4256b4e8&t=event'''); + expect(mockClient.requests[1].buffer.toString(), ''' +Request to https://www.google-analytics.com/batch with ${createUserAgent()} +ec=my-event4&ea=something4&an=usage-test&av=0.0.1&ul=en-us&v=1&tid=%3CTRACKING-ID&cid=8e3fa343-70bc-4afe-ad81-5fed4256b4e8&t=event'''); + expect(mockClient.requests[2].buffer.toString(), ''' +Request to https://www.google-analytics.com/collect with ${createUserAgent()} +ec=my-event-not-batched&ea=something&an=usage-test&av=0.0.1&ul=en-us&v=1&tid=%3CTRACKING-ID&cid=8e3fa343-70bc-4afe-ad81-5fed4256b4e8&t=event'''); + }); + }); } class MockHttpClient implements HttpClient { + final List<MockHttpClientRequest> requests = <MockHttpClientRequest>[]; @override String? userAgent; - int sendCount = 0; - int writeCount = 0; - bool closed = false; + MockHttpClient(); @override - Future<HttpClientRequest> postUrl(Uri url) { - return Future.value(MockHttpClientRequest(this)); + Future<HttpClientRequest> postUrl(Uri uri) async { + final request = MockHttpClientRequest(); + request.buffer.writeln('Request to $uri with $userAgent'); + requests.add(request); + return request; } @override - dynamic noSuchMethod(Invocation invocation) {} + dynamic noSuchMethod(Invocation invocation) { + throw UnimplementedError('Unexpected call'); + } } class MockHttpClientRequest implements HttpClientRequest { - final MockHttpClient client; + final buffer = StringBuffer(); + final MockHttpClientResponse response = MockHttpClientResponse(); - MockHttpClientRequest(this.client); + MockHttpClientRequest(); @override - void write(Object? obj) { - client.writeCount++; + void write(Object? o) { + buffer.write(o); } @override - Future<HttpClientResponse> close() { - client.closed = true; - return Future.value(MockHttpClientResponse(client)); - } + Future<HttpClientResponse> close() async => response; @override - dynamic noSuchMethod(Invocation invocation) {} + dynamic noSuchMethod(Invocation invocation) { + throw UnimplementedError('Unexpected call'); + } } class MockHttpClientResponse implements HttpClientResponse { - final MockHttpClient client; - - MockHttpClientResponse(this.client); + bool drained = false; + MockHttpClientResponse(); @override - Future<E> drain<E>([E? futureValue]) { - client.sendCount++; - return Future.value(); + Future<E> drain<E>([E? futureValue]) async { + drained = true; + return futureValue as E; } @override - dynamic noSuchMethod(Invocation invocation) {} + dynamic noSuchMethod(Invocation invocation) { + throw UnimplementedError('Unexpected call'); + } }
diff --git a/test/web_test.dart b/test/web_test.dart index eb0b107..2a544ba 100644 --- a/test/web_test.dart +++ b/test/web_test.dart
@@ -32,7 +32,9 @@ test('sendPost', () async { var client = MockRequestor(); var postHandler = HtmlPostHandler(mockRequestor: client.request); - var args = <String, dynamic>{'utv': 'varName', 'utt': 123}; + var args = [ + <String, String>{'utv': 'varName', 'utt': '123'}, + ]; await postHandler.sendPost('http://www.google.com', args); expect(client.sendCount, 1);