Retry filesystem deletes (#1965)
We frequently see CI failures with exceptions pointing to various
deletes which fail because of processes still having the file or
directory open. Add a retry with some async backoff.
diff --git a/pkgs/test/lib/src/runner/browser/chrome.dart b/pkgs/test/lib/src/runner/browser/chrome.dart
index 0feb397..9835006 100644
--- a/pkgs/test/lib/src/runner/browser/chrome.dart
+++ b/pkgs/test/lib/src/runner/browser/chrome.dart
@@ -88,8 +88,8 @@
remoteDebuggerCompleter.complete(null);
}
- unawaited(process.exitCode
- .then((_) => Directory(dir).deleteSync(recursive: true)));
+ unawaited(
+ process.exitCode.then((_) => Directory(dir).deleteWithRetry()));
return process;
}
diff --git a/pkgs/test/lib/src/runner/browser/firefox.dart b/pkgs/test/lib/src/runner/browser/firefox.dart
index 5b68018..50fdecf 100644
--- a/pkgs/test/lib/src/runner/browser/firefox.dart
+++ b/pkgs/test/lib/src/runner/browser/firefox.dart
@@ -50,8 +50,7 @@
'MOZ_CRASHREPORTER_DISABLE': '1'
});
- unawaited(process.exitCode
- .then((_) => Directory(dir).deleteSync(recursive: true)));
+ unawaited(process.exitCode.then((_) => Directory(dir).deleteWithRetry()));
return process;
}
diff --git a/pkgs/test/lib/src/runner/browser/platform.dart b/pkgs/test/lib/src/runner/browser/platform.dart
index a44eb7e..80fd0df 100644
--- a/pkgs/test/lib/src/runner/browser/platform.dart
+++ b/pkgs/test/lib/src/runner/browser/platform.dart
@@ -472,7 +472,7 @@
]);
if (_config.pubServeUrl == null) {
- Directory(_compiledDir!).deleteSync(recursive: true);
+ await Directory(_compiledDir!).deleteWithRetry();
} else {
_http!.close();
}
diff --git a/pkgs/test/lib/src/runner/browser/safari.dart b/pkgs/test/lib/src/runner/browser/safari.dart
index 41cd896..6689a8b 100644
--- a/pkgs/test/lib/src/runner/browser/safari.dart
+++ b/pkgs/test/lib/src/runner/browser/safari.dart
@@ -40,8 +40,7 @@
var process = await Process.start(
settings.executable, settings.arguments.toList()..add(redirect));
- unawaited(process.exitCode
- .then((_) => Directory(dir).deleteSync(recursive: true)));
+ unawaited(process.exitCode.then((_) => Directory(dir).deleteWithRetry()));
return process;
}
diff --git a/pkgs/test/lib/src/runner/node/platform.dart b/pkgs/test/lib/src/runner/node/platform.dart
index 8d04f46..23a362a 100644
--- a/pkgs/test/lib/src/runner/node/platform.dart
+++ b/pkgs/test/lib/src/runner/node/platform.dart
@@ -298,7 +298,7 @@
await _compilers.close();
if (_config.pubServeUrl == null) {
- Directory(_compiledDir).deleteSync(recursive: true);
+ await Directory(_compiledDir).deleteWithRetry();
} else {
_http!.close();
}
diff --git a/pkgs/test/lib/src/runner/wasm/platform.dart b/pkgs/test/lib/src/runner/wasm/platform.dart
index 7ed30f5..0103171 100644
--- a/pkgs/test/lib/src/runner/wasm/platform.dart
+++ b/pkgs/test/lib/src/runner/wasm/platform.dart
@@ -358,9 +358,8 @@
browser.then((b) => b?.close()),
_server.close(),
_compilers.close(),
+ Directory(_compiledDir).deleteWithRetry(),
]);
-
- Directory(_compiledDir).deleteSync(recursive: true);
});
final _closeMemo = AsyncMemoizer<void>();
}
diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart
index 75e56dc..c260c9c 100644
--- a/pkgs/test_core/lib/src/runner/vm/platform.dart
+++ b/pkgs/test_core/lib/src/runner/vm/platform.dart
@@ -26,6 +26,7 @@
import '../../runner/plugin/shared_platform_helpers.dart';
import '../../runner/runner_suite.dart';
import '../../runner/suite.dart';
+import '../../util/io.dart';
import '../../util/package_config.dart';
import '../package_version.dart';
import 'environment.dart';
@@ -157,8 +158,10 @@
}
@override
- Future close() => _closeMemo.runOnce(() =>
- Future.wait([_compiler.dispose(), _tempDir.delete(recursive: true)]));
+ Future close() => _closeMemo.runOnce(() => Future.wait([
+ _compiler.dispose(),
+ _tempDir.deleteWithRetry(),
+ ]));
Uri _absolute(String path) {
final uri = p.toUri(path);
diff --git a/pkgs/test_core/lib/src/runner/vm/test_compiler.dart b/pkgs/test_core/lib/src/runner/vm/test_compiler.dart
index a87a70b..7ca295e 100644
--- a/pkgs/test_core/lib/src/runner/vm/test_compiler.dart
+++ b/pkgs/test_core/lib/src/runner/vm/test_compiler.dart
@@ -13,6 +13,7 @@
import 'package:test_api/backend.dart'; // ignore: deprecated_member_use
import '../../util/dart.dart';
+import '../../util/io.dart';
import '../../util/package_config.dart';
import '../package_version.dart';
@@ -182,7 +183,7 @@
_frontendServerClient?.kill();
_frontendServerClient = null;
if (_outputDillDirectory.existsSync()) {
- _outputDillDirectory.deleteSync(recursive: true);
+ await _outputDillDirectory.deleteWithRetry();
}
});
}
diff --git a/pkgs/test_core/lib/src/util/io.dart b/pkgs/test_core/lib/src/util/io.dart
index 2fb0131..59e613c 100644
--- a/pkgs/test_core/lib/src/util/io.dart
+++ b/pkgs/test_core/lib/src/util/io.dart
@@ -7,6 +7,7 @@
import 'dart:core' as core;
import 'dart:core';
import 'dart:io';
+import 'dart:math';
import 'package:async/async.dart';
import 'package:path/path.dart' as p;
@@ -112,7 +113,7 @@
return Future.sync(() {
var tempDir = createTempDir();
return Future.sync(() => fn(tempDir))
- .whenComplete(() => Directory(tempDir).deleteSync(recursive: true));
+ .whenComplete(() => Directory(tempDir).deleteWithRetry());
});
}
@@ -226,3 +227,19 @@
return base;
}
}
+
+extension RetryDelete on FileSystemEntity {
+ Future<void> deleteWithRetry() async {
+ var attempt = 0;
+ while (true) {
+ try {
+ await delete(recursive: true);
+ return;
+ } on FileSystemException {
+ if (attempt == 2) rethrow;
+ attempt++;
+ await Future.delayed(Duration(milliseconds: pow(10, attempt).toInt()));
+ }
+ }
+ }
+}
