Mark package-configs as active when writing (#4592)
diff --git a/doc/cache_layout.md b/doc/cache_layout.md
index 47c6305..3347a2f 100644
--- a/doc/cache_layout.md
+++ b/doc/cache_layout.md
@@ -34,6 +34,7 @@
```plaintext
$PUB_CACHE/
├── global_packages/ # Globally activated packages
+├── active_roots/ # Information about packages that this cache caches for.
├── bin/ # Executables compiled from globally activated packages.
├── git/ # Cloned git packages
├── hosted/ # Hosted package downloads
@@ -233,9 +234,30 @@
└── stagehand
```
+# Active roots
+$PUB_CACHE/active_roots/
+
+In order to be able to prune the cache (`dart pub cache gc`) pub keeps a tally of
+each time it writes a `.dart_tool/package_config.json` file (an activation).
+
+The directory is laid out such that each file-name is the hex-encoded sha256
+hash of the absolute file-uri of the path of the package config.
+
+The first two bytes are used for a subdirectory, to prevent too many files in
+one directory.
+
+When implemented `dart pub cache gc` will look through all the package configs,
+and mark all cached packages in the cache used by those projects `alive`. If a
+package config doesn't exist, it is ignored, and the file marking it is deleted.
+
+All other packages in the cache are removed.
+
+Packages that are installed in the cache within 1 day are not deleted. This is
+to minimize the risk of race-conditions.
+
## Logs
When pub crashes or is run with `--verbose` it will create a
`$PUB_CACHE/log/pub_log.txt` with the dart sdk version, platform, `$PUB_CACHE`,
-`$PUB_HOSTED_URL`, `pubspec.yaml`, `pubspec.lock`, current command, verbose log and
-stack-trace.
+`$PUB_HOSTED_URL`, `pubspec.yaml`, `pubspec.lock`, current command, verbose log
+and stack-trace.
diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart
index 43e3713..1de6570 100644
--- a/lib/src/entrypoint.dart
+++ b/lib/src/entrypoint.dart
@@ -402,6 +402,8 @@
/// If the workspace is non-trivial: For each package in the workspace write:
/// `.dart_tool/pub/workspace_ref.json` with a pointer to the workspace root
/// package dir.
+ ///
+ /// Also marks the package active in `PUB_CACHE/active_roots/`.
Future<void> writePackageConfigFiles() async {
ensureDir(p.dirname(packageConfigPath));
@@ -433,6 +435,9 @@
writeTextFileIfDifferent(workspaceRefPath, '$workspaceRef\n');
}
}
+ if (lockFile.packages.values.any((id) => id.source is CachedSource)) {
+ cache.markRootActive(packageConfigPath);
+ }
}
Future<String> _packageGraphFile(SystemCache cache) async {
diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart
index 9332447..25169eb 100644
--- a/lib/src/global_packages.dart
+++ b/lib/src/global_packages.dart
@@ -293,10 +293,14 @@
lockFile.writeToFile(p.join(tempDir, 'pubspec.lock'), cache);
+ final packageDir = _packageDir(name);
+ tryDeleteEntry(packageDir);
+ tryRenameDir(tempDir, packageDir);
+
// Load the package graph from [result] so we don't need to re-parse all
// the pubspecs.
final entrypoint = Entrypoint.global(
- root,
+ packageForConstraint(dep, packageDir),
lockFile,
cache,
solveResult: result,
@@ -305,9 +309,6 @@
await entrypoint.writePackageConfigFiles();
await entrypoint.precompileExecutables();
-
- tryDeleteEntry(_packageDir(name));
- tryRenameDir(tempDir, _packageDir(name));
}
final entrypoint = Entrypoint.global(
diff --git a/lib/src/io.dart b/lib/src/io.dart
index b7d02ae..fd9f646 100644
--- a/lib/src/io.dart
+++ b/lib/src/io.dart
@@ -257,6 +257,10 @@
File(file).writeAsStringSync(contents, encoding: encoding);
}
+/// Reads the file at [path] and writes [newContent] to it, if it is different
+/// from [newContent].
+///
+/// If the file doesn't exist it is always written.
void writeTextFileIfDifferent(String path, String newContent) {
// Compare to the present package_config.json
// For purposes of equality we don't care about the `generated` timestamp.
diff --git a/lib/src/system_cache.dart b/lib/src/system_cache.dart
index bf52180..90735e7 100644
--- a/lib/src/system_cache.dart
+++ b/lib/src/system_cache.dart
@@ -2,8 +2,10 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
+import 'dart:convert';
import 'dart:io';
+import 'package:crypto/crypto.dart';
import 'package:path/path.dart' as p;
import 'package:pub_semver/pub_semver.dart';
@@ -80,6 +82,8 @@
Source get defaultSource => hosted;
+ late final Iterable<CachedSource> cachedSources = [hosted, git];
+
/// The built-in Git source.
GitSource get git => GitSource.instance;
@@ -400,6 +404,74 @@
}
bool _hasMaintainedCache = false;
+
+ late final _activeRootsDir = p.join(rootDir, 'active_roots');
+
+ /// Returns the paths of all packages_configs registered in
+ /// [_activeRootsDir].
+ List<String> activeRoots() {
+ final List<String> files;
+ try {
+ files = listDir(_activeRootsDir, includeDirs: false, recursive: true);
+ } on IOException {
+ return [];
+ }
+ final activeRoots = <String>[];
+ for (final file in files) {
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(readTextFile(file));
+ } on IOException catch (e) {
+ log.fine('Could not read $file $e - deleting');
+ tryDeleteEntry(file);
+ continue;
+ } on FormatException catch (e) {
+ log.fine('Could not decode $file $e - deleting');
+ tryDeleteEntry(file);
+ continue;
+ }
+ if (decoded is! Map<String, Object?>) {
+ log.fine('Faulty $file - deleting');
+ tryDeleteEntry(file);
+ continue;
+ }
+ final uriText = decoded['package_config'];
+ if (uriText is! String) {
+ log.fine('Faulty $file - deleting');
+ tryDeleteEntry(file);
+ continue;
+ }
+ final uri = Uri.tryParse(uriText);
+ if (uri == null || !uri.isScheme('file')) {
+ log.fine('Faulty $file - deleting');
+ tryDeleteEntry(file);
+ continue;
+ }
+ activeRoots.add(uri.toFilePath());
+ }
+ return activeRoots;
+ }
+
+ /// Adds a file to the `PUB_CACHE/active_roots/` dir indicating
+ /// [packageConfigPath] is active.
+ void markRootActive(String packageConfigPath) {
+ final canonicalFileUri =
+ p.toUri(p.canonicalize(packageConfigPath)).toString();
+
+ final hash = hexEncode(sha256.convert(utf8.encode(canonicalFileUri)).bytes);
+
+ final firstTwo = hash.substring(0, 2);
+ final theRest = hash.substring(2);
+
+ final dir = p.join(_activeRootsDir, firstTwo);
+ ensureDir(dir);
+
+ final filename = p.join(dir, theRest);
+ writeTextFileIfDifferent(
+ filename,
+ '${jsonEncode({'package_config': canonicalFileUri})}\n',
+ );
+ }
}
typedef SourceRegistry = Source Function(String? name);
diff --git a/test/cache/gc_test.dart b/test/cache/gc_test.dart
new file mode 100644
index 0000000..a56944e
--- /dev/null
+++ b/test/cache/gc_test.dart
@@ -0,0 +1,93 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:crypto/crypto.dart';
+import 'package:path/path.dart' as p;
+import 'package:pub/src/system_cache.dart';
+import 'package:pub/src/utils.dart';
+import 'package:test/test.dart';
+
+import '../descriptor.dart' as d;
+import '../test_pub.dart';
+
+void main() async {
+ test('marks a package active on pub get and global activate', () async {
+ final server = await servePackages();
+
+ server.serve('foo', '1.0.0');
+ server.serve('bar', '1.0.0');
+
+ await runPub(args: ['global', 'activate', 'foo']);
+
+ // Without cached dependencies we don't register the package
+ await d.dir('app_none', [d.appPubspec()]).create();
+
+ await d.appDir(dependencies: {'bar': '1.0.0'}).create();
+
+ await d.dir('app_hosted', [
+ d.appPubspec(dependencies: {'bar': '^1.0.0'}),
+ ]).create();
+
+ await d.git('lib', [d.libPubspec('lib', '1.0.0')]).create();
+
+ await d.dir('app_git', [
+ d.appPubspec(
+ dependencies: {
+ 'lib': {'git': '../lib'},
+ },
+ ),
+ ]).create();
+
+ await d.dir('app_path', [
+ d.appPubspec(
+ dependencies: {
+ 'lib': {'path': '../lib'},
+ },
+ ),
+ ]).create();
+
+ await pubGet(workingDirectory: p.join(d.sandbox, 'app_none'));
+ await pubGet(workingDirectory: p.join(d.sandbox, 'app_hosted'));
+ await pubGet(workingDirectory: p.join(d.sandbox, 'app_git'));
+ await pubGet(workingDirectory: p.join(d.sandbox, 'app_path'));
+
+ final markingFiles =
+ Directory(
+ p.join(d.sandbox, cachePath, 'active_roots'),
+ ).listSync(recursive: true).whereType<File>().toList();
+
+ expect(markingFiles, hasLength(3));
+
+ for (final file in markingFiles) {
+ final uri =
+ (jsonDecode(file.readAsStringSync()) as Map)['package_config']
+ as String;
+ final hash = hexEncode(sha256.convert(utf8.encode(uri)).bytes);
+ final hashFileName =
+ '${p.basename(p.dirname(file.path))}${p.basename(file.path)}';
+ expect(hashFileName, hash);
+ }
+
+ expect(markingFiles, hasLength(3));
+
+ expect(SystemCache(rootDir: p.join(d.sandbox, cachePath)).activeRoots(), {
+ p.canonicalize(
+ p.join(d.sandbox, 'app_hosted', '.dart_tool', 'package_config.json'),
+ ),
+ p.canonicalize(
+ p.join(d.sandbox, 'app_git', '.dart_tool', 'package_config.json'),
+ ),
+
+ p.canonicalize(
+ p.join(
+ d.sandbox,
+ cachePath,
+ 'global_packages',
+ 'foo',
+ '.dart_tool',
+ 'package_config.json',
+ ),
+ ),
+ });
+ });
+}
diff --git a/test/embedding/embedding_test.dart b/test/embedding/embedding_test.dart
index db63269..83176d8 100644
--- a/test/embedding/embedding_test.dart
+++ b/test/embedding/embedding_test.dart
@@ -437,7 +437,13 @@
String _filter(String input) {
return input
- .replaceAll(p.toUri(d.sandbox).toString(), r'file://$SANDBOX')
+ .replaceAll(
+ RegExp(
+ RegExp.escape(p.toUri(d.sandbox).toString()),
+ caseSensitive: false,
+ ),
+ r'file://$SANDBOX',
+ )
.replaceAll(d.sandbox, r'$SANDBOX')
.replaceAll(Platform.pathSeparator, '/')
.replaceAll(Platform.operatingSystem, r'$OS')
@@ -546,6 +552,10 @@
RegExp(r'"archive_sha256":"[0-9a-f]{64}"', multiLine: true),
r'"archive_sha256":"$SHA256"',
)
+ .replaceAll(
+ RegExp(r'active_roots/[0-9a-f]{2}/[0-9a-f]{62}', multiLine: true),
+ r'active_roots/$HH/$HASH',
+ )
/// TODO(sigurdm): This hack suppresses differences in stack-traces
/// between dart 2.17 and 2.18. Remove when 2.18 is stable.
.replaceAllMapped(
diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt
index 41f85c0..21b203d 100644
--- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt
+++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt
@@ -149,6 +149,9 @@
[E] | ],
[E] | "configVersion": 1
[E] | }
+[E] IO : Writing $N characters to text file $SANDBOX/cache/active_roots/$HH/$HASH.
+[E] FINE: Contents:
+[E] | {"package_config":"file://$SANDBOX/myapp/.dart_tool/package_config.json"}
[E] IO : Writing $N characters to text file $SANDBOX/cache/log/pub_log.txt.
-------------------------------- END OF OUTPUT ---------------------------------
@@ -335,6 +338,9 @@
| ],
| "configVersion": 1
| }
+IO : Writing $N characters to text file $SANDBOX/cache/active_roots/$HH/$HASH.
+FINE: Contents:
+ | {"package_config":"file://$SANDBOX/myapp/.dart_tool/package_config.json"}
---- End log transcript ----
-------------------------------- END OF OUTPUT ---------------------------------
