Mark package-configs as active when writing (#4592)
diff --git a/doc/cache_layout.md b/doc/cache_layout.md index 47c6305..3347a2f 100644 --- a/doc/cache_layout.md +++ b/doc/cache_layout.md
@@ -34,6 +34,7 @@ ```plaintext $PUB_CACHE/ ├── global_packages/ # Globally activated packages +├── active_roots/ # Information about packages that this cache caches for. ├── bin/ # Executables compiled from globally activated packages. ├── git/ # Cloned git packages ├── hosted/ # Hosted package downloads @@ -233,9 +234,30 @@ └── stagehand ``` +# Active roots +$PUB_CACHE/active_roots/ + +In order to be able to prune the cache (`dart pub cache gc`) pub keeps a tally of +each time it writes a `.dart_tool/package_config.json` file (an activation). + +The directory is laid out such that each file-name is the hex-encoded sha256 +hash of the absolute file-uri of the path of the package config. + +The first two bytes are used for a subdirectory, to prevent too many files in +one directory. + +When implemented `dart pub cache gc` will look through all the package configs, +and mark all cached packages in the cache used by those projects `alive`. If a +package config doesn't exist, it is ignored, and the file marking it is deleted. + +All other packages in the cache are removed. + +Packages that are installed in the cache within 1 day are not deleted. This is +to minimize the risk of race-conditions. + ## Logs When pub crashes or is run with `--verbose` it will create a `$PUB_CACHE/log/pub_log.txt` with the dart sdk version, platform, `$PUB_CACHE`, -`$PUB_HOSTED_URL`, `pubspec.yaml`, `pubspec.lock`, current command, verbose log and -stack-trace. +`$PUB_HOSTED_URL`, `pubspec.yaml`, `pubspec.lock`, current command, verbose log +and stack-trace.
diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index 43e3713..1de6570 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart
@@ -402,6 +402,8 @@ /// If the workspace is non-trivial: For each package in the workspace write: /// `.dart_tool/pub/workspace_ref.json` with a pointer to the workspace root /// package dir. + /// + /// Also marks the package active in `PUB_CACHE/active_roots/`. Future<void> writePackageConfigFiles() async { ensureDir(p.dirname(packageConfigPath)); @@ -433,6 +435,9 @@ writeTextFileIfDifferent(workspaceRefPath, '$workspaceRef\n'); } } + if (lockFile.packages.values.any((id) => id.source is CachedSource)) { + cache.markRootActive(packageConfigPath); + } } Future<String> _packageGraphFile(SystemCache cache) async {
diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart index 9332447..25169eb 100644 --- a/lib/src/global_packages.dart +++ b/lib/src/global_packages.dart
@@ -293,10 +293,14 @@ lockFile.writeToFile(p.join(tempDir, 'pubspec.lock'), cache); + final packageDir = _packageDir(name); + tryDeleteEntry(packageDir); + tryRenameDir(tempDir, packageDir); + // Load the package graph from [result] so we don't need to re-parse all // the pubspecs. final entrypoint = Entrypoint.global( - root, + packageForConstraint(dep, packageDir), lockFile, cache, solveResult: result, @@ -305,9 +309,6 @@ await entrypoint.writePackageConfigFiles(); await entrypoint.precompileExecutables(); - - tryDeleteEntry(_packageDir(name)); - tryRenameDir(tempDir, _packageDir(name)); } final entrypoint = Entrypoint.global(
diff --git a/lib/src/io.dart b/lib/src/io.dart index b7d02ae..fd9f646 100644 --- a/lib/src/io.dart +++ b/lib/src/io.dart
@@ -257,6 +257,10 @@ File(file).writeAsStringSync(contents, encoding: encoding); } +/// Reads the file at [path] and writes [newContent] to it, if it is different +/// from [newContent]. +/// +/// If the file doesn't exist it is always written. void writeTextFileIfDifferent(String path, String newContent) { // Compare to the present package_config.json // For purposes of equality we don't care about the `generated` timestamp.
diff --git a/lib/src/system_cache.dart b/lib/src/system_cache.dart index bf52180..90735e7 100644 --- a/lib/src/system_cache.dart +++ b/lib/src/system_cache.dart
@@ -2,8 +2,10 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; import 'dart:io'; +import 'package:crypto/crypto.dart'; import 'package:path/path.dart' as p; import 'package:pub_semver/pub_semver.dart'; @@ -80,6 +82,8 @@ Source get defaultSource => hosted; + late final Iterable<CachedSource> cachedSources = [hosted, git]; + /// The built-in Git source. GitSource get git => GitSource.instance; @@ -400,6 +404,74 @@ } bool _hasMaintainedCache = false; + + late final _activeRootsDir = p.join(rootDir, 'active_roots'); + + /// Returns the paths of all packages_configs registered in + /// [_activeRootsDir]. + List<String> activeRoots() { + final List<String> files; + try { + files = listDir(_activeRootsDir, includeDirs: false, recursive: true); + } on IOException { + return []; + } + final activeRoots = <String>[]; + for (final file in files) { + final Object? decoded; + try { + decoded = jsonDecode(readTextFile(file)); + } on IOException catch (e) { + log.fine('Could not read $file $e - deleting'); + tryDeleteEntry(file); + continue; + } on FormatException catch (e) { + log.fine('Could not decode $file $e - deleting'); + tryDeleteEntry(file); + continue; + } + if (decoded is! Map<String, Object?>) { + log.fine('Faulty $file - deleting'); + tryDeleteEntry(file); + continue; + } + final uriText = decoded['package_config']; + if (uriText is! String) { + log.fine('Faulty $file - deleting'); + tryDeleteEntry(file); + continue; + } + final uri = Uri.tryParse(uriText); + if (uri == null || !uri.isScheme('file')) { + log.fine('Faulty $file - deleting'); + tryDeleteEntry(file); + continue; + } + activeRoots.add(uri.toFilePath()); + } + return activeRoots; + } + + /// Adds a file to the `PUB_CACHE/active_roots/` dir indicating + /// [packageConfigPath] is active. + void markRootActive(String packageConfigPath) { + final canonicalFileUri = + p.toUri(p.canonicalize(packageConfigPath)).toString(); + + final hash = hexEncode(sha256.convert(utf8.encode(canonicalFileUri)).bytes); + + final firstTwo = hash.substring(0, 2); + final theRest = hash.substring(2); + + final dir = p.join(_activeRootsDir, firstTwo); + ensureDir(dir); + + final filename = p.join(dir, theRest); + writeTextFileIfDifferent( + filename, + '${jsonEncode({'package_config': canonicalFileUri})}\n', + ); + } } typedef SourceRegistry = Source Function(String? name);
diff --git a/test/cache/gc_test.dart b/test/cache/gc_test.dart new file mode 100644 index 0000000..a56944e --- /dev/null +++ b/test/cache/gc_test.dart
@@ -0,0 +1,93 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:path/path.dart' as p; +import 'package:pub/src/system_cache.dart'; +import 'package:pub/src/utils.dart'; +import 'package:test/test.dart'; + +import '../descriptor.dart' as d; +import '../test_pub.dart'; + +void main() async { + test('marks a package active on pub get and global activate', () async { + final server = await servePackages(); + + server.serve('foo', '1.0.0'); + server.serve('bar', '1.0.0'); + + await runPub(args: ['global', 'activate', 'foo']); + + // Without cached dependencies we don't register the package + await d.dir('app_none', [d.appPubspec()]).create(); + + await d.appDir(dependencies: {'bar': '1.0.0'}).create(); + + await d.dir('app_hosted', [ + d.appPubspec(dependencies: {'bar': '^1.0.0'}), + ]).create(); + + await d.git('lib', [d.libPubspec('lib', '1.0.0')]).create(); + + await d.dir('app_git', [ + d.appPubspec( + dependencies: { + 'lib': {'git': '../lib'}, + }, + ), + ]).create(); + + await d.dir('app_path', [ + d.appPubspec( + dependencies: { + 'lib': {'path': '../lib'}, + }, + ), + ]).create(); + + await pubGet(workingDirectory: p.join(d.sandbox, 'app_none')); + await pubGet(workingDirectory: p.join(d.sandbox, 'app_hosted')); + await pubGet(workingDirectory: p.join(d.sandbox, 'app_git')); + await pubGet(workingDirectory: p.join(d.sandbox, 'app_path')); + + final markingFiles = + Directory( + p.join(d.sandbox, cachePath, 'active_roots'), + ).listSync(recursive: true).whereType<File>().toList(); + + expect(markingFiles, hasLength(3)); + + for (final file in markingFiles) { + final uri = + (jsonDecode(file.readAsStringSync()) as Map)['package_config'] + as String; + final hash = hexEncode(sha256.convert(utf8.encode(uri)).bytes); + final hashFileName = + '${p.basename(p.dirname(file.path))}${p.basename(file.path)}'; + expect(hashFileName, hash); + } + + expect(markingFiles, hasLength(3)); + + expect(SystemCache(rootDir: p.join(d.sandbox, cachePath)).activeRoots(), { + p.canonicalize( + p.join(d.sandbox, 'app_hosted', '.dart_tool', 'package_config.json'), + ), + p.canonicalize( + p.join(d.sandbox, 'app_git', '.dart_tool', 'package_config.json'), + ), + + p.canonicalize( + p.join( + d.sandbox, + cachePath, + 'global_packages', + 'foo', + '.dart_tool', + 'package_config.json', + ), + ), + }); + }); +}
diff --git a/test/embedding/embedding_test.dart b/test/embedding/embedding_test.dart index db63269..83176d8 100644 --- a/test/embedding/embedding_test.dart +++ b/test/embedding/embedding_test.dart
@@ -437,7 +437,13 @@ String _filter(String input) { return input - .replaceAll(p.toUri(d.sandbox).toString(), r'file://$SANDBOX') + .replaceAll( + RegExp( + RegExp.escape(p.toUri(d.sandbox).toString()), + caseSensitive: false, + ), + r'file://$SANDBOX', + ) .replaceAll(d.sandbox, r'$SANDBOX') .replaceAll(Platform.pathSeparator, '/') .replaceAll(Platform.operatingSystem, r'$OS') @@ -546,6 +552,10 @@ RegExp(r'"archive_sha256":"[0-9a-f]{64}"', multiLine: true), r'"archive_sha256":"$SHA256"', ) + .replaceAll( + RegExp(r'active_roots/[0-9a-f]{2}/[0-9a-f]{62}', multiLine: true), + r'active_roots/$HH/$HASH', + ) /// TODO(sigurdm): This hack suppresses differences in stack-traces /// between dart 2.17 and 2.18. Remove when 2.18 is stable. .replaceAllMapped(
diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index 41f85c0..21b203d 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt
@@ -149,6 +149,9 @@ [E] | ], [E] | "configVersion": 1 [E] | } +[E] IO : Writing $N characters to text file $SANDBOX/cache/active_roots/$HH/$HASH. +[E] FINE: Contents: +[E] | {"package_config":"file://$SANDBOX/myapp/.dart_tool/package_config.json"} [E] IO : Writing $N characters to text file $SANDBOX/cache/log/pub_log.txt. -------------------------------- END OF OUTPUT --------------------------------- @@ -335,6 +338,9 @@ | ], | "configVersion": 1 | } +IO : Writing $N characters to text file $SANDBOX/cache/active_roots/$HH/$HASH. +FINE: Contents: + | {"package_config":"file://$SANDBOX/myapp/.dart_tool/package_config.json"} ---- End log transcript ---- -------------------------------- END OF OUTPUT ---------------------------------