| // Copyright 2019 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| import 'dart:async'; |
| import 'dart:collection'; |
| import 'dart:typed_data'; |
| |
| import 'package:crypto/crypto.dart'; |
| |
| import '../base/file_system.dart'; |
| import '../convert.dart'; |
| import '../globals.dart'; |
| import 'build_system.dart'; |
| |
| /// An encoded representation of all file hashes. |
| class FileStorage { |
| FileStorage(this.version, this.files); |
| |
| factory FileStorage.fromBuffer(Uint8List buffer) { |
| final Map<String, Object> json = jsonDecode(utf8.decode(buffer)); |
| final int version = json['version']; |
| final List<Object> rawCachedFiles = json['files']; |
| final List<FileHash> cachedFiles = <FileHash>[ |
| for (Map<String, Object> rawFile in rawCachedFiles) |
| FileHash.fromJson(rawFile) |
| ]; |
| return FileStorage(version, cachedFiles); |
| } |
| |
| final int version; |
| final List<FileHash> files; |
| |
| List<int> toBuffer() { |
| final Map<String, Object> json = <String, Object>{ |
| 'version': version, |
| 'files': <Object>[ |
| for (FileHash file in files) |
| file.toJson() |
| ], |
| }; |
| return utf8.encode(jsonEncode(json)); |
| } |
| } |
| |
| /// A stored file hash and path. |
| class FileHash { |
| FileHash(this.path, this.hash); |
| |
| factory FileHash.fromJson(Map<String, Object> json) { |
| return FileHash(json['path'], json['hash']); |
| } |
| |
| final String path; |
| final String hash; |
| |
| Object toJson() { |
| return <String, Object>{ |
| 'path': path, |
| 'hash': hash, |
| }; |
| } |
| } |
| |
| /// A globally accessible cache of file hashes. |
| /// |
| /// In cases where multiple targets read the same source files as inputs, we |
| /// avoid recomputing or storing multiple copies of hashes by delegating |
| /// through this class. All file hashes are held in memory during a build |
| /// operation, and persisted to cache in the root build directory. |
| /// |
| /// The format of the file store is subject to change and not part of its API. |
| /// |
| /// To regenerate the protobuf entries used to construct the cache: |
| /// 1. If not already installed, https://developers.google.com/protocol-buffers/docs/downloads |
| /// 2. pub global active `protoc-gen-dart` |
| /// 3. protoc -I=lib/src/build_system/ --dart_out=lib/src/build_system/ lib/src/build_system/filecache.proto |
| /// 4. Add licenses headers to the newly generated file and check-in. |
| /// |
| /// See also: https://developers.google.com/protocol-buffers/docs/darttutorial |
| // TODO(jonahwilliams): find a better way to clear out old entries, perhaps |
| // track the last access or modification date? |
| class FileHashStore { |
| FileHashStore(this.environment); |
| |
| final Environment environment; |
| final HashMap<String, String> previousHashes = HashMap<String, String>(); |
| final HashMap<String, String> currentHashes = HashMap<String, String>(); |
| |
| // The name of the file which stores the file hashes. |
| static const String _kFileCache = '.filecache'; |
| |
| // The current version of the file cache storage format. |
| static const int _kVersion = 2; |
| |
| /// Read file hashes from disk. |
| void initialize() { |
| printTrace('Initializing file store'); |
| if (!_cacheFile.existsSync()) { |
| return; |
| } |
| final List<int> data = _cacheFile.readAsBytesSync(); |
| FileStorage fileStorage; |
| try { |
| fileStorage = FileStorage.fromBuffer(data); |
| } catch (err) { |
| printTrace('Filestorage format changed'); |
| _cacheFile.deleteSync(); |
| return; |
| } |
| if (fileStorage.version != _kVersion) { |
| _cacheFile.deleteSync(); |
| return; |
| } |
| for (FileHash fileHash in fileStorage.files) { |
| previousHashes[fileHash.path] = fileHash.hash; |
| } |
| printTrace('Done initializing file store'); |
| } |
| |
| /// Persist file hashes to disk. |
| void persist() { |
| printTrace('Persisting file store'); |
| final File file = _cacheFile; |
| if (!file.existsSync()) { |
| file.createSync(recursive: true); |
| } |
| final List<FileHash> fileHashes = <FileHash>[]; |
| for (MapEntry<String, String> entry in currentHashes.entries) { |
| previousHashes[entry.key] = entry.value; |
| } |
| for (MapEntry<String, String> entry in previousHashes.entries) { |
| fileHashes.add(FileHash(entry.key, entry.value)); |
| } |
| final FileStorage fileStorage = FileStorage( |
| _kVersion, |
| fileHashes, |
| ); |
| final Uint8List buffer = fileStorage.toBuffer(); |
| file.writeAsBytesSync(buffer); |
| printTrace('Done persisting file store'); |
| } |
| |
| /// Computes a hash of the provided files and returns a list of entities |
| /// that were dirty. |
| // TODO(jonahwilliams): compare hash performance with md5 tool on macOS and |
| // linux and certutil on Windows, as well as dividing up computation across |
| // isolates. This also related to the current performance issue with checking |
| // APKs before installing them on device. |
| Future<List<File>> hashFiles(List<File> files) async { |
| final List<File> dirty = <File>[]; |
| for (File file in files) { |
| final String absolutePath = file.resolveSymbolicLinksSync(); |
| final String previousHash = previousHashes[absolutePath]; |
| final List<int> bytes = file.readAsBytesSync(); |
| final String currentHash = md5.convert(bytes).toString(); |
| |
| if (currentHash != previousHash) { |
| dirty.add(file); |
| } |
| currentHashes[absolutePath] = currentHash; |
| } |
| return dirty; |
| } |
| |
| File get _cacheFile => environment.buildDir.childFile(_kFileCache); |
| } |