Rate limit the number of gsutil.py subprocesses tools/approve_results.dart spawns
Without any rate limiting, the `tools/approve_results.dart` tool can
render a machine unusable (e.g. when running with
`tools/approve_results.dart -b '*'`).
Unfortunately each gsutil.py subprocess might launch many more
subprocesses on it's own due to us passing the '-m' flag.
Change-Id: I06f4f54fb87a107d83d34205dafe03a8c73d747d
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/106902
Reviewed-by: Jonas Termansen <sortie@google.com>
Auto-Submit: Martin Kustermann <kustermann@google.com>
Commit-Queue: Martin Kustermann <kustermann@google.com>
diff --git a/tools/bots/results.dart b/tools/bots/results.dart
index 65ae36f..1848db5 100644
--- a/tools/bots/results.dart
+++ b/tools/bots/results.dart
@@ -7,6 +7,9 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
+import 'dart:math';
+
+import 'package:pool/pool.dart';
/// The path to the gsutil script.
String gsutilPy;
@@ -18,30 +21,36 @@
const approvedResultsStoragePath =
"gs://dart-test-results-approved-results/builders";
+/// Limit the number of concurrent subprocesses by half the number of cores.
+final gsutilPool = new Pool(max(1, Platform.numberOfProcessors ~/ 2));
+
/// Runs gsutil with the provided [arguments] and returns the standard output.
/// Returns null if the requested URL didn't exist.
Future<String> runGsutil(List<String> arguments) async {
- final processResult = await Process.run(
- "python", [gsutilPy]..addAll(arguments),
- runInShell: Platform.isWindows);
- if (processResult.exitCode != 0) {
- if (processResult.exitCode == 1 &&
- processResult.stderr.contains("No URLs matched") ||
- processResult.stderr.contains("One or more URLs matched no objects")) {
- return null;
+ return gsutilPool.withResource(() async {
+ final processResult = await Process.run(
+ "python", [gsutilPy]..addAll(arguments),
+ runInShell: Platform.isWindows);
+ if (processResult.exitCode != 0) {
+ if (processResult.exitCode == 1 &&
+ processResult.stderr.contains("No URLs matched") ||
+ processResult.stderr
+ .contains("One or more URLs matched no objects")) {
+ return null;
+ }
+ String error = "Failed to run: python $gsutilPy $arguments\n"
+ "exitCode: ${processResult.exitCode}\n"
+ "stdout:\n${processResult.stdout}\n"
+ "stderr:\n${processResult.stderr}";
+ if (processResult.exitCode == 1 &&
+ processResult.stderr.contains("401 Anonymous caller")) {
+ error =
+ "\n\nYou need to authenticate by running:\npython $gsutilPy config\n";
+ }
+ throw new Exception(error);
}
- String error = "Failed to run: python $gsutilPy $arguments\n"
- "exitCode: ${processResult.exitCode}\n"
- "stdout:\n${processResult.stdout}\n"
- "stderr:\n${processResult.stderr}";
- if (processResult.exitCode == 1 &&
- processResult.stderr.contains("401 Anonymous caller")) {
- error =
- "\n\nYou need to authenticate by running:\npython $gsutilPy config\n";
- }
- throw new Exception(error);
- }
- return processResult.stdout;
+ return processResult.stdout;
+ });
}
/// Returns the contents of the provided cloud storage [path], or null if it
