Use the first two key characters for sharding in FileByteStore.
R=brianwilkerson@google.com
Bug: https://github.com/dart-lang/sdk/issues/28383
Change-Id: I177e9bedef79627d7b21e565684004054cef1e54
Reviewed-on: https://dart-review.googlesource.com/c/87521
Commit-Queue: Konstantin Shcheglov <scheglov@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
diff --git a/pkg/analyzer/lib/src/dart/analysis/file_byte_store.dart b/pkg/analyzer/lib/src/dart/analysis/file_byte_store.dart
index 7c1a7fe..1aaa47d 100644
--- a/pkg/analyzer/lib/src/dart/analysis/file_byte_store.dart
+++ b/pkg/analyzer/lib/src/dart/analysis/file_byte_store.dart
@@ -110,7 +110,8 @@
List<File> files = <File>[];
Map<File, FileStat> fileStatMap = {};
int currentSizeBytes = 0;
- List<FileSystemEntity> resources = new Directory(cachePath).listSync();
+ List<FileSystemEntity> resources =
+ new Directory(cachePath).listSync(recursive: true);
for (FileSystemEntity resource in resources) {
if (resource is File) {
try {
@@ -148,6 +149,7 @@
*/
class FileByteStore implements ByteStore {
static final FileByteStoreValidator _validator = new FileByteStoreValidator();
+ static final _dotCodeUnit = '.'.codeUnitAt(0);
final String _cachePath;
final String _tempSuffix;
@@ -164,17 +166,18 @@
@override
List<int> get(String key) {
+ if (!_canShard(key)) return null;
+
List<int> bytes = _writeInProgress[key];
if (bytes != null) {
return bytes;
}
try {
- final File file = _getFileForKey(key);
- if (!file.existsSync()) {
- return null;
- }
- return _validator.getData(file.readAsBytesSync());
+ var shardPath = _getShardPath(key);
+ var path = join(shardPath, key);
+ var bytes = new File(path).readAsBytesSync();
+ return _validator.getData(bytes);
} catch (_) {
// ignore exceptions
return null;
@@ -183,15 +186,22 @@
@override
void put(String key, List<int> bytes) {
+ if (!_canShard(key)) return;
+
_writeInProgress[key] = bytes;
final List<int> wrappedBytes = _validator.wrapData(bytes);
// We don't wait for the write and rename to complete.
_pool.execute(() {
- final File tempFile = _getFileForKey('$key$_tempSuffix');
+ var tempPath = join(_cachePath, '$key$_tempSuffix');
+ var tempFile = new File(tempPath);
return tempFile.writeAsBytes(wrappedBytes).then((_) {
- return tempFile.rename(join(_cachePath, key));
+ var shardPath = _getShardPath(key);
+ return Directory(shardPath).create(recursive: true).then((_) {
+ var path = join(shardPath, key);
+ return tempFile.rename(path);
+ });
}).catchError((_) {
// ignore exceptions
}).whenComplete(() {
@@ -202,7 +212,16 @@
});
}
- File _getFileForKey(String key) => new File(join(_cachePath, key));
+ String _getShardPath(String key) {
+ var shardName = key.substring(0, 2);
+ return join(_cachePath, shardName);
+ }
+
+ static bool _canShard(String key) {
+ return key.length > 2 &&
+ key.codeUnitAt(0) != _dotCodeUnit &&
+ key.codeUnitAt(1) != _dotCodeUnit;
+ }
}
/**
diff --git a/pkg/analyzer/lib/src/dart/analysis/protected_file_byte_store.dart b/pkg/analyzer/lib/src/dart/analysis/protected_file_byte_store.dart
deleted file mode 100644
index b5ec4e1..0000000
--- a/pkg/analyzer/lib/src/dart/analysis/protected_file_byte_store.dart
+++ /dev/null
@@ -1,220 +0,0 @@
-// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-import 'dart:convert';
-import 'dart:io';
-
-import 'package:meta/meta.dart';
-import 'package:path/path.dart';
-
-import 'byte_store.dart';
-import 'cache.dart';
-import 'file_byte_store.dart';
-
-/// The function that returns current time in milliseconds.
-typedef int GetCurrentTime();
-
-/// [ByteStore] that stores values as files, allows to mark some of keys as
-/// temporary protected, and supports periodical [flush] of unprotected keys.
-///
-/// The set of protected keys is stored in a file, which is locked during
-/// updating to prevent races across multiple processes.
-class ProtectedFileByteStore implements ByteStore {
- @visibleForTesting
- static const PROTECTED_FILE_NAME = '.temporary_protected_keys';
-
- final String _cachePath;
- final Duration _protectionDuration;
- final GetCurrentTime _getCurrentTimeFunction;
-
- final FileByteStore _fileByteStore;
- final Cache<String, List<int>> _cache;
-
- /// Create a new instance of the [ProtectedFileByteStore].
- ///
- /// The [protectionDuration] specifies how long temporary protected keys
- /// stay protected.
- ProtectedFileByteStore(this._cachePath,
- {Duration protectionDuration,
- GetCurrentTime getCurrentTime,
- int cacheSizeBytes: 128 * 1024 * 1024})
- : _protectionDuration = protectionDuration,
- _getCurrentTimeFunction = getCurrentTime ?? _getCurrentTimeDefault,
- _fileByteStore = new FileByteStore(_cachePath),
- _cache = new Cache(cacheSizeBytes, (bytes) => bytes.length);
-
- /// Remove all not protected keys.
- void flush() {
- var protectedKeysText = _keysReadTextLocked();
- var protectedKeys = new ProtectedKeys.decode(protectedKeysText);
- List<FileSystemEntity> files = new Directory(_cachePath).listSync();
- for (var file in files) {
- if (file is File) {
- String key = basename(file.path);
- if (key == PROTECTED_FILE_NAME) {
- continue;
- }
- if (protectedKeys.containsKey(key)) {
- continue;
- }
- try {
- file.deleteSync();
- } catch (e) {}
- }
- }
- }
-
- @override
- List<int> get(String key) {
- return _cache.get(key, () => _fileByteStore.get(key));
- }
-
- @override
- void put(String key, List<int> bytes) {
- if (key == PROTECTED_FILE_NAME) {
- throw new ArgumentError('The key $key is reserved.');
- }
- _fileByteStore.put(key, bytes);
- _cache.put(key, bytes);
- }
-
- /// The [add] keys are added to the set of temporary protected keys, and
- /// their age is reset to zero.
- ///
- /// The [remove] keys are removed from the set of temporary protected keys,
- /// and become subjects of LRU cached eviction.
- void updateProtectedKeys(
- {List<String> add: const <String>[],
- List<String> remove: const <String>[]}) {
- _withProtectedKeysLockSync(_cachePath, (ProtectedKeys protectedKeys) {
- var now = _getCurrentTimeFunction();
-
- if (_protectionDuration != null) {
- var maxAge = _protectionDuration.inMilliseconds;
- protectedKeys.removeOlderThan(maxAge, now);
- }
-
- for (var addedKey in add) {
- protectedKeys.add(addedKey, now);
- }
-
- for (var removedKey in remove) {
- protectedKeys.remove(removedKey);
- }
- });
- }
-
- /// Read the protected keys, but don't keep the lock.
- ///
- /// We do this before performing any long running operation that
- /// just read, and where it is important to keep system unlocked.
- String _keysReadTextLocked() {
- File keysFile = new File(join(_cachePath, PROTECTED_FILE_NAME));
- RandomAccessFile keysLock = keysFile.openSync(mode: FileMode.append);
- keysLock.lockSync(FileLock.blockingExclusive);
- try {
- return _keysReadText(keysLock);
- } finally {
- keysLock.unlockSync();
- keysLock.closeSync();
- }
- }
-
- /// The default implementation of [GetCurrentTime].
- static int _getCurrentTimeDefault() {
- return new DateTime.now().millisecondsSinceEpoch;
- }
-
- static ProtectedKeys _keysRead(RandomAccessFile file) {
- String text = _keysReadText(file);
- return new ProtectedKeys.decode(text);
- }
-
- static String _keysReadText(RandomAccessFile file) {
- file.setPositionSync(0);
- List<int> bytes = file.readSync(file.lengthSync());
- return utf8.decode(bytes);
- }
-
- static void _keysWrite(RandomAccessFile file, ProtectedKeys keys) {
- String text = keys.encode();
- file.setPositionSync(0);
- file.writeStringSync(text);
- file.truncateSync(file.positionSync());
- }
-
- /// Perform [f] over the locked keys file, decoded into [ProtectedKeys].
- static void _withProtectedKeysLockSync(
- String cachePath, void f(ProtectedKeys keys)) {
- String path = join(cachePath, PROTECTED_FILE_NAME);
- RandomAccessFile file = new File(path).openSync(mode: FileMode.append);
- file.lockSync(FileLock.blockingExclusive);
- try {
- ProtectedKeys keys = _keysRead(file);
- f(keys);
- _keysWrite(file, keys);
- } finally {
- file.unlockSync();
- file.closeSync();
- }
- }
-}
-
-/// Container with protected keys.
-@visibleForTesting
-class ProtectedKeys {
- /// The map from a key in [ByteStore] to the time in milliseconds when the
- /// key was marked as temporary protected.
- final Map<String, int> map;
-
- ProtectedKeys(this.map);
-
- factory ProtectedKeys.decode(String text) {
- var map = <String, int>{};
- try {
- List<String> lines = text.split('\n').toList();
- if (lines.length % 2 == 0) {
- for (int i = 0; i < lines.length; i += 2) {
- String key = lines[i];
- String startMillisecondsStr = lines[i + 1];
- int startMilliseconds = int.parse(startMillisecondsStr);
- map[key] = startMilliseconds;
- }
- }
- } catch (e) {}
- return new ProtectedKeys(map);
- }
-
- /// Add the given [key] with the current time.
- void add(String key, int time) {
- map[key] = time;
- }
-
- bool containsKey(String key) => map.containsKey(key);
-
- String encode() {
- var buffer = new StringBuffer();
- map.forEach((key, start) {
- buffer.writeln(key);
- buffer.writeln(start);
- });
- return buffer.toString().trim();
- }
-
- void remove(String key) {
- map.remove(key);
- }
-
- /// If the time is [now] milliseconds, remove all keys that are older than
- /// the given [maxAge] is milliseconds.
- void removeOlderThan(int maxAge, int now) {
- var keysToRemove = <String>[];
- for (var key in map.keys) {
- if (now - map[key] > maxAge) {
- keysToRemove.add(key);
- }
- }
- keysToRemove.forEach(map.remove);
- }
-}
diff --git a/pkg/analyzer/test/src/dart/analysis/protected_file_byte_store_test.dart b/pkg/analyzer/test/src/dart/analysis/protected_file_byte_store_test.dart
deleted file mode 100644
index 05bcad5..0000000
--- a/pkg/analyzer/test/src/dart/analysis/protected_file_byte_store_test.dart
+++ /dev/null
@@ -1,269 +0,0 @@
-// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-import 'dart:async';
-import 'dart:io' as io;
-
-import 'package:analyzer/src/dart/analysis/protected_file_byte_store.dart';
-import 'package:path/path.dart' as pathos;
-import 'package:test/test.dart';
-import 'package:test_reflective_loader/test_reflective_loader.dart';
-
-main() {
- defineReflectiveSuite(() {
- defineReflectiveTests(ProtectedKeysTest);
- defineReflectiveTests(ProtectedFileByteStoreTest);
- });
-}
-
-List<int> _b(int length) {
- return new List<int>.filled(length, 0);
-}
-
-@reflectiveTest
-class ProtectedFileByteStoreTest {
- static const PADDING = 4;
-
- io.Directory cacheDirectory;
- String cachePath;
- ProtectedFileByteStore store;
-
- int time = 0;
-
- String get protectedKeysText {
- String path =
- pathos.join(cachePath, ProtectedFileByteStore.PROTECTED_FILE_NAME);
- return new io.File(path).readAsStringSync();
- }
-
- void setUp() {
- io.Directory systemTemp = io.Directory.systemTemp;
- cacheDirectory = systemTemp.createTempSync('ProtectedFileByteStoreTest');
- cachePath = cacheDirectory.absolute.path;
- _createStore();
- }
-
- void tearDown() {
- try {
- cacheDirectory.deleteSync(recursive: true);
- } on io.FileSystemException {}
- }
-
- test_flush() async {
- store.put('a', _b(1));
- store.put('b', _b(2));
- store.put('c', _b(3));
- store.put('d', _b(4));
-
- store.updateProtectedKeys(add: ['b', 'd']);
-
- // Add a delay to give the store time to write to disk.
- await new Future.delayed(const Duration(milliseconds: 200));
-
- // Flush, only protected 'b' and 'd' survive.
- store.flush();
- store.flush();
- _assertCacheContent({'b': 2, 'd': 4}, ['a', 'c']);
-
- // Remove 'b' and flush.
- // Only 'd' survives.
- store.updateProtectedKeys(remove: ['b']);
- store.flush();
- _assertCacheContent({'d': 4}, ['b']);
- }
-
- test_put() async {
- store.put('a', _b(65));
- store.put('b', _b(63));
- store.put('c', _b(1));
-
- // We can access all results.
- expect(store.get('a'), hasLength(65));
- expect(store.get('b'), hasLength(63));
- expect(store.get('c'), hasLength(1));
-
- // Add a delay to give the store time to write to disk.
- await new Future.delayed(const Duration(milliseconds: 200));
-
- _assertCacheContent({'a': 65, 'b': 63, 'c': 1}, []);
- }
-
- test_put_reservedKey() {
- expect(() {
- store.put(ProtectedFileByteStore.PROTECTED_FILE_NAME, <int>[]);
- }, throwsArgumentError);
- }
-
- test_updateProtectedKeys_add() {
- store.updateProtectedKeys(add: ['a', 'b']);
- _assertKeys({'a': 0, 'b': 0});
-
- time++;
- store.updateProtectedKeys(add: ['c']);
- _assertKeys({'a': 0, 'b': 0, 'c': 1});
- }
-
- test_updateProtectedKeys_add_hasSame() {
- store.updateProtectedKeys(add: ['a', 'b', 'c']);
- _assertKeys({'a': 0, 'b': 0, 'c': 0});
-
- time++;
- store.updateProtectedKeys(add: ['b', 'd']);
- _assertKeys({'a': 0, 'b': 1, 'c': 0, 'd': 1});
- }
-
- test_updateProtectedKeys_add_removeTooOld() {
- store.updateProtectedKeys(add: ['a', 'b']);
- _assertKeys({'a': 0, 'b': 0});
-
- // Move time to 10 ms, both 'a' and 'b' are still alive.
- time = 10;
- store.updateProtectedKeys(add: ['c']);
- _assertKeys({'a': 0, 'b': 0, 'c': 10});
-
- // Move time to 11 ms, now 'a' and 'b' are too old and removed.
- time = 11;
- store.updateProtectedKeys(add: ['d']);
- _assertKeys({'c': 10, 'd': 11});
- }
-
- test_updateProtectedKeys_add_removeTooOld_nullDuration() {
- _createStore(protectionDuration: null);
-
- store.updateProtectedKeys(add: ['a', 'b']);
- _assertKeys({'a': 0, 'b': 0});
-
- // Move time far into the future, both 'a' and 'b' are still alive.
- time = 1 << 30;
- store.updateProtectedKeys(add: ['c']);
- _assertKeys({'a': 0, 'b': 0, 'c': time});
- }
-
- test_updateProtectedKeys_addRemove() {
- store.updateProtectedKeys(add: ['a', 'b', 'c']);
- _assertKeys({'a': 0, 'b': 0, 'c': 0});
-
- time++;
- store.updateProtectedKeys(add: ['d'], remove: ['b']);
- _assertKeys({'a': 0, 'c': 0, 'd': 1});
- }
-
- test_updateProtectedKeys_addRemove_same() {
- store.updateProtectedKeys(add: ['a', 'b', 'c']);
- _assertKeys({'a': 0, 'b': 0, 'c': 0});
-
- time++;
- store.updateProtectedKeys(add: ['b'], remove: ['b']);
- _assertKeys({'a': 0, 'c': 0});
- }
-
- test_updateProtectedKeys_remove() {
- store.updateProtectedKeys(add: ['a', 'b', 'c']);
- _assertKeys({'a': 0, 'b': 0, 'c': 0});
-
- time++;
- store.updateProtectedKeys(remove: ['b']);
- _assertKeys({'a': 0, 'c': 0});
- }
-
- void _assertCacheContent(Map<String, int> includes, List<String> excludes) {
- Map<String, int> keyToLength = {};
- for (var file in cacheDirectory.listSync()) {
- String key = pathos.basename(file.path);
- if (file is io.File) {
- keyToLength[key] = file.lengthSync();
- }
- }
- includes.forEach((expectedKey, expectedLength) {
- expect(keyToLength, contains(expectedKey));
- expect(keyToLength, containsPair(expectedKey, expectedLength + PADDING));
- });
- for (var excludedKey in excludes) {
- expect(keyToLength.keys, isNot(contains(excludedKey)));
- }
- }
-
- void _assertKeys(Map<String, int> expected) {
- var path =
- pathos.join(cachePath, ProtectedFileByteStore.PROTECTED_FILE_NAME);
- var text = new io.File(path).readAsStringSync();
- var keys = new ProtectedKeys.decode(text);
- expect(keys.map.keys, expected.keys);
- expected.forEach((key, start) {
- expect(keys.map, containsPair(key, start));
- });
- }
-
- void _createStore(
- {Duration protectionDuration: const Duration(milliseconds: 10)}) {
- store = new ProtectedFileByteStore(cachePath,
- protectionDuration: protectionDuration,
- cacheSizeBytes: 256,
- getCurrentTime: _getTime);
- }
-
- int _getTime() => time;
-}
-
-@reflectiveTest
-class ProtectedKeysTest {
- test_decode() {
- var keys = new ProtectedKeys({'/a/b/c': 10, '/a/d/e': 123});
-
- String text = keys.encode();
- expect(text, r'''
-/a/b/c
-10
-/a/d/e
-123''');
-
- keys = _decode(text);
- expect(keys.map['/a/b/c'], 10);
- expect(keys.map['/a/d/e'], 123);
- }
-
- test_decode_empty() {
- var keys = _decode('');
- expect(keys.map, isEmpty);
- }
-
- test_decode_error_notEvenNumberOfLines() {
- var keys = _decode('a');
- expect(keys.map, isEmpty);
- }
-
- test_decode_error_startIsEmpty() {
- var keys = _decode('a\n');
- expect(keys.map, isEmpty);
- }
-
- test_decode_error_startIsNotInt() {
- var keys = _decode('a\n1.23');
- expect(keys.map, isEmpty);
- }
-
- test_decode_error_startIsNotNumber() {
- var keys = _decode('a\nb');
- expect(keys.map, isEmpty);
- }
-
- test_removeOlderThan() {
- var keys = new ProtectedKeys({'a': 1, 'b': 2, 'c': 3});
- _assertKeys(keys, {'a': 1, 'b': 2, 'c': 3});
-
- keys.removeOlderThan(5, 7);
- _assertKeys(keys, {'b': 2, 'c': 3});
- }
-
- void _assertKeys(ProtectedKeys keys, Map<String, int> expected) {
- expect(keys.map.keys, expected.keys);
- expected.forEach((key, start) {
- expect(keys.map, containsPair(key, start));
- });
- }
-
- ProtectedKeys _decode(String text) {
- return new ProtectedKeys.decode(text);
- }
-}
diff --git a/pkg/analyzer/test/src/dart/analysis/test_all.dart b/pkg/analyzer/test/src/dart/analysis/test_all.dart
index 8cda60d..77d968e 100644
--- a/pkg/analyzer/test/src/dart/analysis/test_all.dart
+++ b/pkg/analyzer/test/src/dart/analysis/test_all.dart
@@ -21,7 +21,6 @@
import 'fletcher16_test.dart' as fletcher16_test;
import 'index_test.dart' as index;
import 'mutex_test.dart' as mutex;
-import 'protected_file_byte_store_test.dart' as protected_file_byte_store_test;
import 'referenced_names_test.dart' as referenced_names;
import 'search_test.dart' as search;
import 'session_helper_test.dart' as session_helper;
@@ -47,7 +46,6 @@
fletcher16_test.main();
index.main();
mutex.main();
- protected_file_byte_store_test.main();
referenced_names.main();
search.main();
session.main();
