blob: b52e80accb563ee32c767b6c3f4654311dfd0d61 [file] [log] [blame]
// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'dart:math' show max, sqrt;
import 'package:logging/logging.dart';
import 'package:path/path.dart';
import '../../test/integration/support/integration_test_methods.dart';
import '../../test/integration/support/integration_tests.dart';
import 'operation.dart';
final SPACE = ' '.codeUnitAt(0);
void _printColumn(StringBuffer sb, String text, int keyLen,
{bool rightJustified = false}) {
if (!rightJustified) {
sb.write(text);
sb.write(',');
}
for (var i = text.length; i < keyLen; ++i) {
sb.writeCharCode(SPACE);
}
if (rightJustified) {
sb.write(text);
sb.write(',');
}
sb.writeCharCode(SPACE);
}
/// [Driver] launches and manages an instance of analysis server,
/// reads a stream of operations, sends requests to analysis server
/// based upon those operations, and evaluates the results.
class Driver extends IntegrationTestMixin {
/// The amount of time to give the server to respond to a shutdown request
/// before forcibly terminating it.
static const Duration SHUTDOWN_TIMEOUT = Duration(seconds: 5);
final Logger logger = Logger('Driver');
/// The diagnostic port for Analysis Server or `null` if none.
final int? diagnosticPort;
/// A flag indicating whether the server is running.
bool running = false;
@override
late Server server;
/// The results collected while running analysis server.
final Results results = Results();
/// The [Completer] for [runComplete].
final Completer<Results> _runCompleter = Completer<Results>();
Driver({this.diagnosticPort});
/// Return a [Future] that completes with the [Results] of running
/// the analysis server once all operations have been performed.
Future<Results> get runComplete => _runCompleter.future;
/// Perform the given operation.
/// Return a [Future] that completes when the next operation can be performed,
/// or `null` if the next operation can be performed immediately
Future<void>? perform(Operation op) {
return op.perform(this);
}
/// Send a command to the server. An 'id' will be automatically assigned.
/// The returned [Future] will be completed when the server acknowledges the
/// command with a response. If the server acknowledges the command with a
/// normal (non-error) response, the future will be completed with the
/// 'result' field from the response. If the server acknowledges the command
/// with an error response, the future will be completed with an error.
Future<Map<String, Object?>?> send(
String method, Map<String, dynamic> params) {
return server.send(method, params);
}
/// Launch the analysis server.
/// Return a [Future] that completes when analysis server has started.
Future startServer() async {
logger.log(Level.FINE, 'starting server');
initializeInttestMixin();
server = Server();
var serverConnected = Completer();
onServerConnected.listen((_) {
logger.log(Level.FINE, 'connected to server');
serverConnected.complete();
});
running = true;
var dartSdkPath = dirname(dirname(Platform.resolvedExecutable));
return server
.start(dartSdkPath: dartSdkPath, diagnosticPort: diagnosticPort)
.then((params) {
server.listenToOutput(dispatchNotification);
server.exitCode.then((_) {
logger.log(Level.FINE, 'server stopped');
running = false;
_resultsReady();
});
return serverConnected.future;
});
}
/// Shutdown the analysis server if it is running.
Future stopServer([Duration timeout = SHUTDOWN_TIMEOUT]) async {
if (running) {
logger.log(Level.FINE, 'requesting server shutdown');
// Give the server a short time to comply with the shutdown request; if it
// doesn't exit, then forcibly terminate it.
sendServerShutdown();
await server.exitCode.timeout(timeout, onTimeout: () {
return server.kill('server failed to exit');
});
}
_resultsReady();
}
/// If not already complete, signal the completer with the collected results.
void _resultsReady() {
if (!_runCompleter.isCompleted) {
_runCompleter.complete(results);
}
}
}
/// [Measurement] tracks elapsed time for a given operation.
class Measurement {
final String tag;
final bool notification;
final List<Duration> elapsedTimes = <Duration>[];
int errorCount = 0;
int unexpectedResultCount = 0;
Measurement(this.tag, this.notification);
int get count => elapsedTimes.length;
void printSummary(int keyLen) {
var count = 0;
var maxTime = elapsedTimes[0];
var minTime = elapsedTimes[0];
var totalTimeMicros = 0;
for (var elapsed in elapsedTimes) {
++count;
var timeMicros = elapsed.inMicroseconds;
maxTime = maxTime.compareTo(elapsed) > 0 ? maxTime : elapsed;
minTime = minTime.compareTo(elapsed) < 0 ? minTime : elapsed;
totalTimeMicros += timeMicros;
}
var meanTime = (totalTimeMicros / count).round();
var sorted = elapsedTimes.toList()..sort();
var time90th = sorted[(sorted.length * 0.90).round() - 1];
var time99th = sorted[(sorted.length * 0.99).round() - 1];
var differenceFromMeanSquared = 0;
for (var elapsed in elapsedTimes) {
var timeMicros = elapsed.inMicroseconds;
var differenceFromMean = timeMicros - meanTime;
differenceFromMeanSquared += differenceFromMean * differenceFromMean;
}
var variance = differenceFromMeanSquared / count;
var standardDeviation = sqrt(variance).round();
var sb = StringBuffer();
_printColumn(sb, tag, keyLen);
_printColumn(sb, count.toString(), 6, rightJustified: true);
_printColumn(sb, errorCount.toString(), 6, rightJustified: true);
_printColumn(sb, unexpectedResultCount.toString(), 6, rightJustified: true);
_printDuration(sb, Duration(microseconds: meanTime));
_printDuration(sb, time90th);
_printDuration(sb, time99th);
_printDuration(sb, Duration(microseconds: standardDeviation));
_printDuration(sb, minTime);
_printDuration(sb, maxTime);
_printDuration(sb, Duration(microseconds: totalTimeMicros));
print(sb.toString());
}
void record(bool success, Duration elapsed) {
if (!success) {
++errorCount;
}
elapsedTimes.add(elapsed);
}
void recordUnexpectedResults() {
++unexpectedResultCount;
}
void _printDuration(StringBuffer sb, Duration duration) {
_printColumn(sb, duration.inMilliseconds.toString(), 15,
rightJustified: true);
}
}
/// [Results] contains information gathered by [Driver]
/// while running the analysis server
class Results {
Map<String, Measurement> measurements = <String, Measurement>{};
/// Display results on stdout.
void printResults() {
print('');
print('==================================================================');
print('');
var sortedEntries = measurements.entries.toList();
sortedEntries.sort((a, b) => a.key.compareTo(b.key));
var keyLen = sortedEntries
.map((e) => e.key)
.fold(0, (int len, String key) => max(len, key.length));
_printGroupHeader('Request/Response', keyLen);
var totalCount = 0;
var totalErrorCount = 0;
var totalUnexpectedResultCount = 0;
for (var entry in sortedEntries) {
var m = entry.value;
if (!m.notification) {
m.printSummary(keyLen);
totalCount += m.count;
totalErrorCount += m.errorCount;
totalUnexpectedResultCount += m.unexpectedResultCount;
}
}
_printTotals(
keyLen, totalCount, totalErrorCount, totalUnexpectedResultCount);
print('');
_printGroupHeader('Notifications', keyLen);
for (var entry in sortedEntries) {
var m = entry.value;
if (m.notification) {
m.printSummary(keyLen);
}
}
/// TODO(danrubel) *** print warnings if driver caches are not empty ****
print('''
(1) uxr = UneXpected Results or responses received from the server
that do not match the recorded response for that request.
(2) all times in milliseconds''');
}
/// Record the elapsed time for the given operation.
void record(String tag, Duration elapsed,
{bool notification = false, bool success = true}) {
var measurement = measurements[tag];
if (measurement == null) {
measurement = Measurement(tag, notification);
measurements[tag] = measurement;
}
measurement.record(success, elapsed);
}
void recordUnexpectedResults(String tag) {
measurements[tag]!.recordUnexpectedResults();
}
void _printGroupHeader(String groupName, int keyLen) {
var sb = StringBuffer();
_printColumn(sb, groupName, keyLen);
_printColumn(sb, 'count', 6, rightJustified: true);
_printColumn(sb, 'error', 6, rightJustified: true);
_printColumn(sb, 'uxr(1)', 6, rightJustified: true);
sb.write(' ');
_printColumn(sb, 'mean(2)', 15);
_printColumn(sb, '90th', 15);
_printColumn(sb, '99th', 15);
_printColumn(sb, 'std-dev', 15);
_printColumn(sb, 'minimum', 15);
_printColumn(sb, 'maximum', 15);
_printColumn(sb, 'total', 15);
print(sb.toString());
}
void _printTotals(int keyLen, int totalCount, int totalErrorCount,
int totalUnexpectedResultCount) {
var sb = StringBuffer();
_printColumn(sb, 'Totals', keyLen);
_printColumn(sb, totalCount.toString(), 6, rightJustified: true);
_printColumn(sb, totalErrorCount.toString(), 6, rightJustified: true);
_printColumn(sb, totalUnexpectedResultCount.toString(), 6,
rightJustified: true);
print(sb.toString());
}
}