new command `pub cache gc` (#4684)
diff --git a/lib/src/command/cache.dart b/lib/src/command/cache.dart index 4a370f7..8ec1f92 100644 --- a/lib/src/command/cache.dart +++ b/lib/src/command/cache.dart
@@ -5,6 +5,7 @@ import '../command.dart'; import 'cache_add.dart'; import 'cache_clean.dart'; +import 'cache_gc.dart'; import 'cache_list.dart'; import 'cache_preload.dart'; import 'cache_repair.dart'; @@ -24,5 +25,6 @@ addSubcommand(CacheCleanCommand()); addSubcommand(CacheRepairCommand()); addSubcommand(CachePreloadCommand()); + addSubcommand(CacheGcCommand()); } }
diff --git a/lib/src/command/cache_gc.dart b/lib/src/command/cache_gc.dart new file mode 100644 index 0000000..0887625 --- /dev/null +++ b/lib/src/command/cache_gc.dart
@@ -0,0 +1,163 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; + +import '../command.dart'; +import '../command_runner.dart'; +import '../io.dart'; +import '../log.dart' as log; +import '../package_config.dart'; +import '../utils.dart'; + +class CacheGcCommand extends PubCommand { + @override + String get name => 'gc'; + @override + String get description => 'Prunes unused packages from the system cache.'; + @override + bool get takesArguments => false; + + final dontRemoveFilesOlderThan = const Duration(hours: 2); + + CacheGcCommand() { + argParser.addFlag( + 'force', + abbr: 'f', + help: 'Prune cache without confirmation', + hideNegatedUsage: true, + ); + argParser.addFlag( + 'collect-recent', + help: 'Also delete recent files', + hideNegatedUsage: true, + ); + argParser.addFlag( + 'dry-run', + help: 'Print list of files that would be deleted', + hideNegatedUsage: true, + ); + } + + @override + Future<void> runProtected() async { + final dryRun = argResults.flag('dry-run'); + final activeRoots = cache.activeRoots(); + // All the `activeRoots` that we could read and parse a + // .dart_tool/packageConfig.json from. + final validActiveRoots = <String>[]; + // All the rootUri paths to cached packages included from + // `validActiveRoots`. + final paths = <String>{}; + for (final packageConfigPath in activeRoots) { + late final PackageConfig packageConfig; + try { + packageConfig = PackageConfig.fromJson( + json.decode(readTextFile(packageConfigPath)), + ); + } on IOException catch (e) { + // Failed to read file - probably got deleted. + log.fine('Failed to read packageConfig $packageConfigPath: $e'); + continue; + } on FormatException catch (e) { + log.warning( + 'Failed to decode packageConfig $packageConfigPath: $e.\n' + 'It could be corrupted', + ); + // Failed to decode - probably corrupted. + continue; + } + for (final package in packageConfig.packages) { + final rootUri = p.canonicalize( + package.resolvedRootDir(packageConfigPath), + ); + if (p.isWithin(cache.rootDir, rootUri)) { + paths.add(rootUri); + } + } + validActiveRoots.add(packageConfigPath); + } + final now = DateTime.now(); + final allPathsToGC = + [ + for (final source in cache.cachedSources) + ...await source.entriesToGc( + cache, + paths + .where( + (path) => p.isWithin( + p.canonicalize(cache.rootDirForSource(source)), + path, + ), + ) + .toSet(), + ), + ].where((path) { + // Only clear cache entries older than 2 hours to avoid race + // conditions with ongoing `pub get` processes. + final s = statPath(path); + if (s.type == FileSystemEntityType.notFound) return false; + if (argResults.flag('collect-recent')) return true; + return now.difference(s.modified) > dontRemoveFilesOlderThan; + }).toList(); + if (validActiveRoots.isEmpty) { + log.message('Found no active projects.'); + } else { + final s = validActiveRoots.length == 1 ? '' : 's'; + log.message('Found ${validActiveRoots.length} active project$s:'); + for (final packageConfigPath in validActiveRoots) { + final parts = p.split(packageConfigPath); + var projectDir = packageConfigPath; + if (parts[parts.length - 2] == '.dart_tool' && + parts[parts.length - 1] == 'package_config.json') { + projectDir = p.joinAll(parts.sublist(0, parts.length - 2)); + } + log.message('* $projectDir'); + } + } + var sum = 0; + for (final entry in allPathsToGC) { + if (dirExists(entry)) { + for (final file in listDir( + entry, + recursive: true, + includeHidden: true, + includeDirs: false, + )) { + sum += tryStatFile(file)?.size ?? 0; + } + } else { + sum += tryStatFile(entry)?.size ?? 0; + } + } + if (sum == 0) { + log.message('No unused cache entries found.'); + return; + } + log.message(''); + log.message( + ''' +All other projects ${dryRun ? 'would' : 'will'} need to run `$topLevelProgram pub get` again to work correctly.''', + ); + log.message( + '${dryRun ? 'Would' : 'Will'} recover ${readableFileSize(sum)}.', + ); + if (dryRun) { + log.message('Would delete:'); + for (final path in allPathsToGC..sort()) { + log.message(path); + } + } else if (argResults.flag('force') || + await confirm('Are you sure you want to continue?')) { + await log.progress('Deleting unused cache entries', () async { + for (final path in allPathsToGC..sort()) { + tryDeleteEntry(path); + } + }); + } + } +}
diff --git a/lib/src/command/lish.dart b/lib/src/command/lish.dart index b2adb58..abd1bfc 100644 --- a/lib/src/command/lish.dart +++ b/lib/src/command/lish.dart
@@ -368,7 +368,7 @@ baseDir: entrypoint.workPackage.dir, ).toBytes(); - final size = _readableFileSize(packageBytes.length); + final size = readableFileSize(packageBytes.length); log.message('\nTotal compressed archive size: $size.\n'); final validationResult = @@ -536,18 +536,6 @@ } } -String _readableFileSize(int size) { - if (size >= 1 << 30) { - return '${size ~/ (1 << 30)} GB'; - } else if (size >= 1 << 20) { - return '${size ~/ (1 << 20)} MB'; - } else if (size >= 1 << 10) { - return '${size ~/ (1 << 10)} KB'; - } else { - return '<1 KB'; - } -} - class _Publication { Uint8List packageBytes; int warningCount;
diff --git a/lib/src/source/cached.dart b/lib/src/source/cached.dart index 2cfa2a1..533009b 100644 --- a/lib/src/source/cached.dart +++ b/lib/src/source/cached.dart
@@ -76,6 +76,14 @@ /// Returns a list of results indicating for each if that package was /// successfully repaired. Future<Iterable<RepairResult>> repairCachedPackages(SystemCache cache); + + /// Return all directories inside this source that can be removed while + /// preserving the packages given by [alivePackages] a list of package root + /// directories. They should all be canonicalized. + Future<List<String>> entriesToGc( + SystemCache cache, + Set<String> alivePackages, + ); } /// The result of repairing a single cache entry.
diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index 9087944..f6ed7b0 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart
@@ -900,6 +900,74 @@ } return name; } + + @override + Future<List<String>> entriesToGc( + SystemCache cache, + Set<String> alivePackages, + ) async { + final rootDir = p.canonicalize(cache.rootDirForSource(this)); + if (!entryExists(rootDir)) return const []; + + final gitDirsToRemove = <String>{}; + // First enumerate all git repos inside [rootDir]. + for (final entry in listDir(rootDir)) { + final gitEntry = p.join(entry, '.git'); + if (!entryExists(gitEntry)) continue; + gitDirsToRemove.add(p.canonicalize(entry)); + } + final cacheDirsToRemove = <String>{}; + try { + cacheDirsToRemove.addAll( + listDir(p.join(rootDir, 'cache')).map(p.canonicalize), + ); + } on IOException { + // Most likely the directory didn't exist. + // ignore. + } + // For each package walk up parent directories to find the containing git + // repo, and mark it alive by removing from `gitDirsToRemove`. + for (final alivePackage in alivePackages) { + var candidate = p.canonicalize(alivePackage); + while (!p.equals(candidate, rootDir)) { + if (gitDirsToRemove.remove(candidate)) { + // Package is alive, now also retain its cachedir. + // + // TODO(sigurdm): Should we just GC all cache-dirs? They are not + // needed for consuming packages, and most likely will be recreated + // when needed. + final gitEntry = p.join(candidate, '.git'); + try { + if (dirExists(gitEntry)) { + final path = + (await git.run([ + 'remote', + 'get-url', + 'origin', + ], workingDir: candidate)).split('\n').first; + cacheDirsToRemove.remove(p.canonicalize(path)); + } else if (fileExists(gitEntry)) { + // Potential future - using worktrees. + final path = + (await git.run([ + 'worktree', + 'list', + '--porcelain', + ], workingDir: candidate)).split('\n').first.split(' ').last; + cacheDirsToRemove.remove(p.canonicalize(path)); + } + } on git.GitException catch (e) { + log.fine('Failed to find canonical cache for $candidate, $e'); + } + break; + } + // Try the parent directory + candidate = p.dirname(candidate); + } + } + + return [...cacheDirsToRemove, ...gitDirsToRemove]; + } } class GitDescription extends Description {
diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index f0be1f9..c3fded5 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart
@@ -1765,6 +1765,62 @@ } } + @override + Future<List<String>> entriesToGc( + SystemCache cache, + Set<String> alivePackages, + ) async { + final root = p.canonicalize(cache.rootDirForSource(this)); + final result = <String>{}; + final List<String> hostDirs; + + try { + hostDirs = listDir(root); + } on IOException { + // Hosted cache seems uninitialized. GC nothing. + return []; + } + for (final hostDir in hostDirs) { + final List<String> packageDirs; + try { + packageDirs = listDir(hostDir).map(p.canonicalize).toList(); + } on IOException { + // Failed to list `hostDir`. Perhaps a stray file? Skip. + continue; + } + for (final packageDir in packageDirs) { + if (!alivePackages.contains(packageDir)) { + result.add(packageDir); + // Also clear the associated hash file. + final hashFile = p.join( + cache.rootDir, + 'hosted-hashes', + p.basename(hostDir), + '${p.basename(packageDir)}.sha256', + ); + if (fileExists(hashFile)) { + result.add(hashFile); + } + } + } + // Clear all version listings older than two days, they'd likely need to + // be re-fetched anyways: + for (final cacheFile in listDir( + p.join(hostDir, _versionListingDirectory), + )) { + final stat = tryStatFile(cacheFile); + + if (stat != null && + DateTime.now().difference(stat.modified) > + const Duration(days: 2)) { + result.add(cacheFile); + } + } + } + + return result.toList(); + } + /// Enables speculative prefetching of dependencies of packages queried with /// [doGetVersions]. Future<T> withPrefetching<T>(Future<T> Function() callback) async {
diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 2a62a94..a769cf6 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart
@@ -825,3 +825,15 @@ ), ]; } + +String readableFileSize(int size) { + if (size >= 1 << 30) { + return '${size ~/ (1 << 30)} GB'; + } else if (size >= 1 << 20) { + return '${size ~/ (1 << 20)} MB'; + } else if (size >= 1 << 10) { + return '${size ~/ (1 << 10)} KB'; + } else { + return '<1 KB'; + } +}
diff --git a/pubspec.yaml b/pubspec.yaml index d3fb28b..87b5444 100644 --- a/pubspec.yaml +++ b/pubspec.yaml
@@ -19,7 +19,7 @@ http_parser: ^4.1.2 meta: ^1.17.0 path: ^1.9.1 - pool: ^1.5.1 + pool: ^1.0.0 pub_semver: ^2.2.0 shelf: ^1.4.2 source_span: ^1.10.1
diff --git a/test/cache/gc_test.dart b/test/cache/gc_test.dart index a56944e..e6116d4 100644 --- a/test/cache/gc_test.dart +++ b/test/cache/gc_test.dart
@@ -90,4 +90,130 @@ ), }); }); + + test('gcing an empty cache behaves well', () async { + await runPub( + args: ['cache', 'gc', '--force', '--collect-recent'], + output: allOf( + contains('Found no active projects.'), + contains('No unused cache entries found.'), + ), + ); + }); + + test('can gc cache entries', () async { + final server = await servePackages(); + + server.serve('hosted1', '1.0.0'); + server.serve('hosted2', '1.0.0'); + + await d.git('git1', [d.libPubspec('git1', '1.0.0')]).create(); + await d.git('git2', [d.libPubspec('git2', '1.0.0')]).create(); + + await d.git('git_with_path1', [ + d.dir('pkg', [d.libPubspec('git_with_path1', '1.0.0')]), + ]).create(); + await d.git('git_with_path2', [ + d.dir('pkg', [d.libPubspec('git_with_path2', '1.0.0')]), + ]).create(); + + await d + .appDir( + dependencies: { + 'hosted1': '1.0.0', + 'git1': {'git': '../git1'}, + 'git_with_path1': { + 'git': {'url': '../git_with_path1', 'path': 'pkg'}, + }, + }, + ) + .create(); + await pubGet(); + await d + .appDir( + dependencies: { + 'hosted2': '1.0.0', + 'git2': {'git': '../git2'}, + 'git_with_path2': { + 'git': {'url': '../git_with_path2', 'path': 'pkg'}, + }, + }, + ) + .create(); + await pubGet(output: contains('- hosted1')); + final matchesAppPath = matches( + RegExp( + RegExp.escape('* ${p.join(d.sandbox, appPath)}'), + caseSensitive: false, + ), + ); + // First cache gc run finds nothing because the entries are recent. + await runPub( + args: ['cache', 'gc', '--force'], + output: allOf(matchesAppPath, contains('No unused cache entries found')), + ); + final s = RegExp.escape(p.separator); + await runPub( + args: ['cache', 'gc', '--force', '--collect-recent', '--dry-run'], + output: allOf([ + matchesAppPath, + contains(RegExp('Would recover [0-9]{3} KB.')), + contains(RegExp('.*git.*cache${s}git1-.*')), + contains(RegExp('.*git.*cache${s}git_with_path1-.*')), + contains(RegExp('.*git.*git1-.*')), + contains(RegExp('.*git.*git_with_path1-.*')), + contains(RegExp('.*hosted-hashes.*hosted1-1.0.0.sha256')), + contains(RegExp('.*hosted.*hosted1-1.0.0.')), + isNot(contains(RegExp('.*hosted2'))), + isNot(contains(RegExp('.*git2'))), + isNot(contains(RegExp('.*git_with_path2'))), + ]), + ); + await runPub( + args: ['cache', 'gc', '--force', '--collect-recent'], + output: allOf( + matchesAppPath, + contains(RegExp('Will recover [0-9]{3} KB.')), + ), + silent: allOf([ + contains(RegExp('Deleting directory .*git.*cache${s}git1-.*')), + contains( + RegExp('Deleting directory .*git.*cache${s}git_with_path1-.*'), + ), + contains(RegExp('Deleting directory .*git.*git1-.*')), + contains(RegExp('Deleting directory .*git.*git_with_path1-.*')), + contains(RegExp('Deleting file .*hosted-hashes.*hosted1-1.0.0.sha256')), + contains(RegExp('Deleting directory .*hosted.*hosted1-1.0.0.')), + isNot(contains(RegExp('Deleting.*hosted2'))), + isNot(contains(RegExp('Deleting.*git2'))), + isNot(contains(RegExp('Deleting.*git_with_path2'))), + ]), + ); + expect( + Directory( + p.join(d.sandbox, d.hostedCachePath(), 'hosted1-1.0.0'), + ).existsSync(), + isFalse, + ); + expect( + Directory( + p.join(d.sandbox, d.hostedCachePath(), 'hosted2-1.0.0'), + ).existsSync(), + isTrue, + ); + + expect( + Directory( + p.join(d.sandbox, cachePath, 'git'), + ).listSync().map((f) => p.basename(f.path)), + {'cache', matches(RegExp('git2.*')), matches(RegExp('git_with_path2.*'))}, + ); + + expect( + Directory( + p.join(d.sandbox, cachePath, 'git', 'cache'), + ).listSync().map((f) => p.basename(f.path)), + {matches(RegExp('git2.*')), matches(RegExp('git_with_path2.*'))}, + ); + }); }
diff --git a/test/testdata/goldens/help_test/pub cache --help.txt b/test/testdata/goldens/help_test/pub cache --help.txt index fdb0b2c..d9a6427 100644 --- a/test/testdata/goldens/help_test/pub cache --help.txt +++ b/test/testdata/goldens/help_test/pub cache --help.txt
@@ -10,6 +10,7 @@ Available subcommands: add Install a package. clean Clears the global PUB_CACHE. + gc Prunes unused packages from the system cache. repair Reinstall cached packages. Run "pub help" to see global options.
diff --git a/test/testdata/goldens/help_test/pub cache gc --help.txt b/test/testdata/goldens/help_test/pub cache gc --help.txt new file mode 100644 index 0000000..4c132a3 --- /dev/null +++ b/test/testdata/goldens/help_test/pub cache gc --help.txt
@@ -0,0 +1,14 @@ +# GENERATED BY: test/help_test.dart + +## Section 0 +$ pub cache gc --help +Prunes unused packages from the system cache. + +Usage: pub cache gc <subcommand> [arguments...] +-h, --help Print this usage information. +-f, --force Prune cache without confirmation + --collect-recent Also delete recent files + --dry-run Print list of files that would be deleted + +Run "pub help" to see global options. +