Add failure reasons to tool call analytics events (#219)
This should help us to understand why tool calls fail, instead of just seeing that they failed.
In particular for tests as an example, we can discard the ones that fail due to a non-zero exit code.
diff --git a/pkgs/dart_mcp_server/CHANGELOG.md b/pkgs/dart_mcp_server/CHANGELOG.md
index 9ae564a..5c1088e 100644
--- a/pkgs/dart_mcp_server/CHANGELOG.md
+++ b/pkgs/dart_mcp_server/CHANGELOG.md
@@ -2,6 +2,8 @@
* Change tools that accept multiple roots to not return immediately on the first
failure.
+* Add failure reason field to analytics events so we can know why tool calls are
+ failing.
# 0.1.0 (Dart SDK 3.9.0)
diff --git a/pkgs/dart_mcp_server/lib/src/mixins/analyzer.dart b/pkgs/dart_mcp_server/lib/src/mixins/analyzer.dart
index eb0f2fe..610ae70 100644
--- a/pkgs/dart_mcp_server/lib/src/mixins/analyzer.dart
+++ b/pkgs/dart_mcp_server/lib/src/mixins/analyzer.dart
@@ -13,6 +13,7 @@
import 'package:meta/meta.dart';
import '../lsp/wire_format.dart';
+import '../utils/analytics.dart';
import '../utils/constants.dart';
import '../utils/sdk.dart';
@@ -474,7 +475,7 @@
'tool.',
),
],
- );
+ )..failureReason = CallToolFailureReason.noRootsSet;
}
/// Common schema for tools that require a file URI, line, and column.
diff --git a/pkgs/dart_mcp_server/lib/src/mixins/dash_cli.dart b/pkgs/dart_mcp_server/lib/src/mixins/dash_cli.dart
index cf137ad..64a5bf7 100644
--- a/pkgs/dart_mcp_server/lib/src/mixins/dash_cli.dart
+++ b/pkgs/dart_mcp_server/lib/src/mixins/dash_cli.dart
@@ -8,6 +8,7 @@
import 'package:dart_mcp/server.dart';
import 'package:path/path.dart' as p;
+import '../utils/analytics.dart';
import '../utils/cli_utils.dart';
import '../utils/constants.dart';
import '../utils/file_system.dart';
@@ -141,7 +142,7 @@
for (final error in errors) Content.text(text: error.toErrorString()),
],
isError: true,
- );
+ )..failureReason = CallToolFailureReason.argumentError;
}
final template = args[ParameterNames.template] as String?;
diff --git a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart
index 591f3ab..89f2ab1 100644
--- a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart
+++ b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart
@@ -199,7 +199,7 @@
text: 'Connection failed, make sure your DTD Uri is up to date.',
),
],
- );
+ )..failureReason = CallToolFailureReason.webSocketException;
} catch (e) {
return CallToolResult(
isError: true,
@@ -532,7 +532,7 @@
'boolean.',
),
],
- );
+ )..failureReason = CallToolFailureReason.argumentError;
}
return _callOnVmService(
@@ -750,7 +750,7 @@
'connect to Dart and Flutter applications.',
),
],
- );
+ )..failureReason = CallToolFailureReason.connectedAppServiceNotSupported;
static final _dtdNotConnected = CallToolResult(
isError: true,
@@ -761,7 +761,7 @@
'"${connectTool.name}" first.',
),
],
- );
+ )..failureReason = CallToolFailureReason.dtdNotConnected;
static final _dtdAlreadyConnected = CallToolResult(
isError: true,
@@ -772,14 +772,14 @@
'"${connectTool.name}" again.',
),
],
- );
+ )..failureReason = CallToolFailureReason.dtdAlreadyConnected;
static final _noActiveDebugSession = CallToolResult(
content: [
TextContent(text: 'No active debug session to take a screenshot'),
],
isError: true,
- );
+ )..failureReason = CallToolFailureReason.noActiveDebugSession;
static final runtimeErrorsScheme = 'runtime-errors';
}
diff --git a/pkgs/dart_mcp_server/lib/src/mixins/pub.dart b/pkgs/dart_mcp_server/lib/src/mixins/pub.dart
index 9059a10..31abce9 100644
--- a/pkgs/dart_mcp_server/lib/src/mixins/pub.dart
+++ b/pkgs/dart_mcp_server/lib/src/mixins/pub.dart
@@ -6,6 +6,7 @@
import 'package:dart_mcp/server.dart';
+import '../utils/analytics.dart';
import '../utils/cli_utils.dart';
import '../utils/constants.dart';
import '../utils/file_system.dart';
@@ -47,7 +48,7 @@
),
],
isError: true,
- );
+ )..failureReason ??= CallToolFailureReason.noSuchCommand;
}
final packageName =
@@ -62,7 +63,7 @@
),
],
isError: true,
- );
+ )..failureReason ??= CallToolFailureReason.argumentError;
}
return runCommandInRoots(
diff --git a/pkgs/dart_mcp_server/lib/src/mixins/pub_dev_search.dart b/pkgs/dart_mcp_server/lib/src/mixins/pub_dev_search.dart
index fc5c83e..e164203 100644
--- a/pkgs/dart_mcp_server/lib/src/mixins/pub_dev_search.dart
+++ b/pkgs/dart_mcp_server/lib/src/mixins/pub_dev_search.dart
@@ -9,6 +9,7 @@
import 'package:http/http.dart';
import 'package:pool/pool.dart';
+import '../utils/analytics.dart';
import '../utils/json.dart';
/// Limit the number of concurrent requests.
@@ -36,7 +37,7 @@
return CallToolResult(
content: [TextContent(text: 'Missing required argument `query`.')],
isError: true,
- );
+ )..failureReason = CallToolFailureReason.argumentError;
}
final searchUrl = Uri.https('pub.dev', 'api/search', {'q': query});
final Object? result;
@@ -54,7 +55,6 @@
text: 'No packages matched the query, consider simplifying it.',
),
],
- isError: true,
);
}
diff --git a/pkgs/dart_mcp_server/lib/src/server.dart b/pkgs/dart_mcp_server/lib/src/server.dart
index 27374ed..4eb854b 100644
--- a/pkgs/dart_mcp_server/lib/src/server.dart
+++ b/pkgs/dart_mcp_server/lib/src/server.dart
@@ -198,6 +198,7 @@
tool: request.name,
success: result != null && result.isError != true,
elapsedMilliseconds: watch.elapsedMilliseconds,
+ failureReason: result?.failureReason,
),
),
);
diff --git a/pkgs/dart_mcp_server/lib/src/utils/analytics.dart b/pkgs/dart_mcp_server/lib/src/utils/analytics.dart
index 9c71dd9..f61c81f 100644
--- a/pkgs/dart_mcp_server/lib/src/utils/analytics.dart
+++ b/pkgs/dart_mcp_server/lib/src/utils/analytics.dart
@@ -2,6 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
+import 'package:dart_mcp/server.dart';
import 'package:unified_analytics/unified_analytics.dart';
/// An interface class that provides a access to an [Analytics] instance, if
@@ -53,10 +54,14 @@
/// The time it took to invoke the tool.
final int elapsedMilliseconds;
+ /// The reason for the failure, if [success] is `false`.
+ final CallToolFailureReason? failureReason;
+
CallToolMetrics({
required this.tool,
required this.success,
required this.elapsedMilliseconds,
+ required this.failureReason,
});
@override
@@ -64,12 +69,42 @@
_tool: tool,
_success: success,
_elapsedMilliseconds: elapsedMilliseconds,
+ _failureReason: ?failureReason?.name,
};
}
enum ResourceKind { runtimeErrors }
+/// Extension which tracks failure reasons for [CallToolResult] objects in an
+/// [Expando].
+extension WithFailureReason on CallToolResult {
+ static final _expando = Expando<CallToolFailureReason>();
+
+ CallToolFailureReason? get failureReason => _expando[this as Object];
+
+ set failureReason(CallToolFailureReason? value) =>
+ _expando[this as Object] = value;
+}
+
+/// Known reasons for failed tool calls.
+enum CallToolFailureReason {
+ argumentError,
+ connectedAppServiceNotSupported,
+ dtdAlreadyConnected,
+ dtdNotConnected,
+ invalidPath,
+ invalidRootPath,
+ invalidRootScheme,
+ noActiveDebugSession,
+ noRootGiven,
+ noRootsSet,
+ noSuchCommand,
+ nonZeroExitCode,
+ webSocketException,
+}
+
const _elapsedMilliseconds = 'elapsedMilliseconds';
+const _failureReason = 'failureReason';
const _kind = 'kind';
const _length = 'length';
const _success = 'success';
diff --git a/pkgs/dart_mcp_server/lib/src/utils/cli_utils.dart b/pkgs/dart_mcp_server/lib/src/utils/cli_utils.dart
index af74733..4b69d94 100644
--- a/pkgs/dart_mcp_server/lib/src/utils/cli_utils.dart
+++ b/pkgs/dart_mcp_server/lib/src/utils/cli_utils.dart
@@ -10,6 +10,7 @@
import 'package:process/process.dart';
import 'package:yaml/yaml.dart';
+import 'analytics.dart';
import 'constants.dart';
import 'sdk.dart';
@@ -159,7 +160,7 @@
TextContent(text: 'Invalid root configuration: missing `root` key.'),
],
isError: true,
- );
+ )..failureReason ??= CallToolFailureReason.noRootGiven;
}
final root = knownRoots.firstWhereOrNull(
@@ -175,7 +176,7 @@
),
],
isError: true,
- );
+ )..failureReason ??= CallToolFailureReason.invalidRootPath;
}
final rootUri = Uri.parse(rootUriString);
@@ -189,7 +190,7 @@
),
],
isError: true,
- );
+ )..failureReason ??= CallToolFailureReason.invalidRootScheme;
}
final projectRoot = fileSystem.directory(rootUri);
@@ -213,7 +214,7 @@
),
],
isError: true,
- );
+ )..failureReason ??= CallToolFailureReason.invalidPath;
}
commandWithPaths.addAll(paths);
@@ -238,7 +239,7 @@
),
],
isError: true,
- );
+ )..failureReason ??= CallToolFailureReason.nonZeroExitCode;
}
return CallToolResult(
content: [
diff --git a/pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart b/pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart
index 2857be0..0c703b4 100644
--- a/pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart
+++ b/pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart
@@ -7,6 +7,7 @@
import 'package:dart_mcp/server.dart';
import 'package:dart_mcp_server/src/server.dart';
+import 'package:dart_mcp_server/src/utils/analytics.dart';
import 'package:test/test.dart';
import 'package:test_descriptor/test_descriptor.dart' as d;
import 'package:unified_analytics/testing.dart';
@@ -56,32 +57,43 @@
});
test('sends analytics for failed tool calls', () async {
- server.registerTool(
- Tool(name: 'hello', inputSchema: Schema.object()),
- (_) => CallToolResult(isError: true, content: []),
- );
- final result = await testHarness.mcpServerConnection.callTool(
- CallToolRequest(name: 'hello'),
- );
- expect(result.isError, true);
- expect(
- analytics.sentEvents.single,
- isA<Event>()
- .having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
- .having(
- (e) => e.eventData,
- 'eventData',
- equals({
- 'client': server.clientInfo.name,
- 'clientVersion': server.clientInfo.version,
- 'serverVersion': server.implementation.version,
- 'type': 'callTool',
- 'tool': 'hello',
- 'success': false,
- 'elapsedMilliseconds': isA<int>(),
- }),
- ),
- );
+ for (var reason in [null, CallToolFailureReason.nonZeroExitCode]) {
+ analytics.sentEvents.clear();
+
+ final tool = Tool(
+ name: 'hello${reason?.name ?? ''}',
+ inputSchema: Schema.object(),
+ );
+ server.registerTool(
+ tool,
+ (_) =>
+ CallToolResult(isError: true, content: [])
+ ..failureReason = reason,
+ );
+ final result = await testHarness.mcpServerConnection.callTool(
+ CallToolRequest(name: tool.name),
+ );
+ expect(result.isError, true);
+ expect(
+ analytics.sentEvents.single,
+ isA<Event>()
+ .having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
+ .having(
+ (e) => e.eventData,
+ 'eventData',
+ equals({
+ 'client': server.clientInfo.name,
+ 'clientVersion': server.clientInfo.version,
+ 'serverVersion': server.implementation.version,
+ 'type': 'callTool',
+ 'tool': tool.name,
+ 'success': false,
+ 'elapsedMilliseconds': isA<int>(),
+ 'failureReason': ?reason?.name,
+ }),
+ ),
+ );
+ }
});
test('Changelog version matches dart server version', () {
diff --git a/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart b/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart
index 436ac6c..ad4e00f 100644
--- a/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart
+++ b/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart
@@ -175,7 +175,7 @@
);
});
- test('No matching packages gets reported as an error', () async {
+ test('No matching packages gets special handling', () async {
await runWithClient(
() async {
await runWithHarness((testHarness, pubDevSearchTool) async {
@@ -187,9 +187,8 @@
final result = await testHarness.callToolWithRetry(
request,
maxTries: 1,
- expectError: true,
);
- expect(result.isError, isTrue);
+ expect(result.isError, isNot(true));
expect(
(result.content[0] as TextContent).text,
contains('No packages matched the query, consider simplifying it'),
