Append version and CRC32 to data in FileByteStore.
We need this to prevent reading accidentally damaged data.
R=brianwilkerson@google.com, paulberry@google.com
Bug:
Change-Id: I1c3b2c61f3cdb087c8f7effafba7fa13deef87ce
Reviewed-on: https://dart-review.googlesource.com/3700
Reviewed-by: Paul Berry <paulberry@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
diff --git a/pkg/analyzer/lib/src/dart/analysis/driver.dart b/pkg/analyzer/lib/src/dart/analysis/driver.dart
index bf98050..0d4084c 100644
--- a/pkg/analyzer/lib/src/dart/analysis/driver.dart
+++ b/pkg/analyzer/lib/src/dart/analysis/driver.dart
@@ -95,7 +95,7 @@
/**
* The version of data format, should be incremented on every format change.
*/
- static const int DATA_VERSION = 58;
+ static const int DATA_VERSION = 59;
/**
* The number of exception contexts allowed to write. Once this field is
diff --git a/pkg/front_end/lib/src/byte_store/crc32.dart b/pkg/front_end/lib/src/byte_store/crc32.dart
new file mode 100644
index 0000000..457712e
--- /dev/null
+++ b/pkg/front_end/lib/src/byte_store/crc32.dart
@@ -0,0 +1,288 @@
+// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+const List<int> _CRC32_TABLE = const [
+ 0x00000000,
+ 0x77073096,
+ 0xEE0E612C,
+ 0x990951BA,
+ 0x076DC419,
+ 0x706AF48F,
+ 0xE963A535,
+ 0x9E6495A3,
+ 0x0EDB8832,
+ 0x79DCB8A4,
+ 0xE0D5E91E,
+ 0x97D2D988,
+ 0x09B64C2B,
+ 0x7EB17CBD,
+ 0xE7B82D07,
+ 0x90BF1D91,
+ 0x1DB71064,
+ 0x6AB020F2,
+ 0xF3B97148,
+ 0x84BE41DE,
+ 0x1ADAD47D,
+ 0x6DDDE4EB,
+ 0xF4D4B551,
+ 0x83D385C7,
+ 0x136C9856,
+ 0x646BA8C0,
+ 0xFD62F97A,
+ 0x8A65C9EC,
+ 0x14015C4F,
+ 0x63066CD9,
+ 0xFA0F3D63,
+ 0x8D080DF5,
+ 0x3B6E20C8,
+ 0x4C69105E,
+ 0xD56041E4,
+ 0xA2677172,
+ 0x3C03E4D1,
+ 0x4B04D447,
+ 0xD20D85FD,
+ 0xA50AB56B,
+ 0x35B5A8FA,
+ 0x42B2986C,
+ 0xDBBBC9D6,
+ 0xACBCF940,
+ 0x32D86CE3,
+ 0x45DF5C75,
+ 0xDCD60DCF,
+ 0xABD13D59,
+ 0x26D930AC,
+ 0x51DE003A,
+ 0xC8D75180,
+ 0xBFD06116,
+ 0x21B4F4B5,
+ 0x56B3C423,
+ 0xCFBA9599,
+ 0xB8BDA50F,
+ 0x2802B89E,
+ 0x5F058808,
+ 0xC60CD9B2,
+ 0xB10BE924,
+ 0x2F6F7C87,
+ 0x58684C11,
+ 0xC1611DAB,
+ 0xB6662D3D,
+ 0x76DC4190,
+ 0x01DB7106,
+ 0x98D220BC,
+ 0xEFD5102A,
+ 0x71B18589,
+ 0x06B6B51F,
+ 0x9FBFE4A5,
+ 0xE8B8D433,
+ 0x7807C9A2,
+ 0x0F00F934,
+ 0x9609A88E,
+ 0xE10E9818,
+ 0x7F6A0DBB,
+ 0x086D3D2D,
+ 0x91646C97,
+ 0xE6635C01,
+ 0x6B6B51F4,
+ 0x1C6C6162,
+ 0x856530D8,
+ 0xF262004E,
+ 0x6C0695ED,
+ 0x1B01A57B,
+ 0x8208F4C1,
+ 0xF50FC457,
+ 0x65B0D9C6,
+ 0x12B7E950,
+ 0x8BBEB8EA,
+ 0xFCB9887C,
+ 0x62DD1DDF,
+ 0x15DA2D49,
+ 0x8CD37CF3,
+ 0xFBD44C65,
+ 0x4DB26158,
+ 0x3AB551CE,
+ 0xA3BC0074,
+ 0xD4BB30E2,
+ 0x4ADFA541,
+ 0x3DD895D7,
+ 0xA4D1C46D,
+ 0xD3D6F4FB,
+ 0x4369E96A,
+ 0x346ED9FC,
+ 0xAD678846,
+ 0xDA60B8D0,
+ 0x44042D73,
+ 0x33031DE5,
+ 0xAA0A4C5F,
+ 0xDD0D7CC9,
+ 0x5005713C,
+ 0x270241AA,
+ 0xBE0B1010,
+ 0xC90C2086,
+ 0x5768B525,
+ 0x206F85B3,
+ 0xB966D409,
+ 0xCE61E49F,
+ 0x5EDEF90E,
+ 0x29D9C998,
+ 0xB0D09822,
+ 0xC7D7A8B4,
+ 0x59B33D17,
+ 0x2EB40D81,
+ 0xB7BD5C3B,
+ 0xC0BA6CAD,
+ 0xEDB88320,
+ 0x9ABFB3B6,
+ 0x03B6E20C,
+ 0x74B1D29A,
+ 0xEAD54739,
+ 0x9DD277AF,
+ 0x04DB2615,
+ 0x73DC1683,
+ 0xE3630B12,
+ 0x94643B84,
+ 0x0D6D6A3E,
+ 0x7A6A5AA8,
+ 0xE40ECF0B,
+ 0x9309FF9D,
+ 0x0A00AE27,
+ 0x7D079EB1,
+ 0xF00F9344,
+ 0x8708A3D2,
+ 0x1E01F268,
+ 0x6906C2FE,
+ 0xF762575D,
+ 0x806567CB,
+ 0x196C3671,
+ 0x6E6B06E7,
+ 0xFED41B76,
+ 0x89D32BE0,
+ 0x10DA7A5A,
+ 0x67DD4ACC,
+ 0xF9B9DF6F,
+ 0x8EBEEFF9,
+ 0x17B7BE43,
+ 0x60B08ED5,
+ 0xD6D6A3E8,
+ 0xA1D1937E,
+ 0x38D8C2C4,
+ 0x4FDFF252,
+ 0xD1BB67F1,
+ 0xA6BC5767,
+ 0x3FB506DD,
+ 0x48B2364B,
+ 0xD80D2BDA,
+ 0xAF0A1B4C,
+ 0x36034AF6,
+ 0x41047A60,
+ 0xDF60EFC3,
+ 0xA867DF55,
+ 0x316E8EEF,
+ 0x4669BE79,
+ 0xCB61B38C,
+ 0xBC66831A,
+ 0x256FD2A0,
+ 0x5268E236,
+ 0xCC0C7795,
+ 0xBB0B4703,
+ 0x220216B9,
+ 0x5505262F,
+ 0xC5BA3BBE,
+ 0xB2BD0B28,
+ 0x2BB45A92,
+ 0x5CB36A04,
+ 0xC2D7FFA7,
+ 0xB5D0CF31,
+ 0x2CD99E8B,
+ 0x5BDEAE1D,
+ 0x9B64C2B0,
+ 0xEC63F226,
+ 0x756AA39C,
+ 0x026D930A,
+ 0x9C0906A9,
+ 0xEB0E363F,
+ 0x72076785,
+ 0x05005713,
+ 0x95BF4A82,
+ 0xE2B87A14,
+ 0x7BB12BAE,
+ 0x0CB61B38,
+ 0x92D28E9B,
+ 0xE5D5BE0D,
+ 0x7CDCEFB7,
+ 0x0BDBDF21,
+ 0x86D3D2D4,
+ 0xF1D4E242,
+ 0x68DDB3F8,
+ 0x1FDA836E,
+ 0x81BE16CD,
+ 0xF6B9265B,
+ 0x6FB077E1,
+ 0x18B74777,
+ 0x88085AE6,
+ 0xFF0F6A70,
+ 0x66063BCA,
+ 0x11010B5C,
+ 0x8F659EFF,
+ 0xF862AE69,
+ 0x616BFFD3,
+ 0x166CCF45,
+ 0xA00AE278,
+ 0xD70DD2EE,
+ 0x4E048354,
+ 0x3903B3C2,
+ 0xA7672661,
+ 0xD06016F7,
+ 0x4969474D,
+ 0x3E6E77DB,
+ 0xAED16A4A,
+ 0xD9D65ADC,
+ 0x40DF0B66,
+ 0x37D83BF0,
+ 0xA9BCAE53,
+ 0xDEBB9EC5,
+ 0x47B2CF7F,
+ 0x30B5FFE9,
+ 0xBDBDF21C,
+ 0xCABAC28A,
+ 0x53B39330,
+ 0x24B4A3A6,
+ 0xBAD03605,
+ 0xCDD70693,
+ 0x54DE5729,
+ 0x23D967BF,
+ 0xB3667A2E,
+ 0xC4614AB8,
+ 0x5D681B02,
+ 0x2A6F2B94,
+ 0xB40BBE37,
+ 0xC30C8EA1,
+ 0x5A05DF1B,
+ 0x2D02EF8D
+];
+
+/**
+ * Get the CRC-32 checksum of the given array. You can append bytes to an
+ * already computed crc by specifying the previous [crc] value.
+ */
+int getCrc32(List<int> array, [int crc = 0]) {
+ int len = array.length;
+ crc = crc ^ 0xffffffff;
+ int ip = 0;
+ while (len >= 8) {
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ len -= 8;
+ }
+ if (len > 0)
+ do {
+ crc = _CRC32_TABLE[(crc ^ array[ip++]) & 0xff] ^ (crc >> 8);
+ } while (--len > 0);
+ return crc ^ 0xffffffff;
+}
diff --git a/pkg/front_end/lib/src/byte_store/file_byte_store.dart b/pkg/front_end/lib/src/byte_store/file_byte_store.dart
index 8ce63de..9a9bc29 100644
--- a/pkg/front_end/lib/src/byte_store/file_byte_store.dart
+++ b/pkg/front_end/lib/src/byte_store/file_byte_store.dart
@@ -5,8 +5,10 @@
import 'dart:async';
import 'dart:io';
import 'dart:isolate';
+import 'dart:typed_data';
import 'package:front_end/src/byte_store/byte_store.dart';
+import 'package:front_end/src/byte_store/crc32.dart';
import 'package:path/path.dart';
/**
@@ -144,6 +146,7 @@
class FileByteStore implements ByteStore {
final String _cachePath;
final String _tempName;
+ final FileByteStoreValidator _validator = new FileByteStoreValidator();
/**
* If the same cache path is used from more than one isolate of the same
@@ -155,7 +158,9 @@
@override
List<int> get(String key) {
try {
- return _getFileForKey(key).readAsBytesSync();
+ File file = _getFileForKey(key);
+ List<int> rawBytes = file.readAsBytesSync();
+ return _validator.getData(rawBytes);
} catch (_) {
return null;
}
@@ -164,6 +169,7 @@
@override
void put(String key, List<int> bytes) {
try {
+ bytes = _validator.wrapData(bytes);
File tempFile = _getFileForKey(_tempName);
tempFile.writeAsBytesSync(bytes);
File file = _getFileForKey(key);
@@ -175,3 +181,75 @@
return new File(join(_cachePath, key));
}
}
+
+/**
+ * Generally speaking, we cannot guarantee that any data written into a
+ * file will stay the same - there is always a chance of a hardware problem,
+ * file system problem, truncated data, etc.
+ *
+ * So, we need to embed some validation into data itself.
+ * This class append the version and the checksum to data.
+ */
+class FileByteStoreValidator {
+ static const List<int> _VERSION = const [0x01, 0x00, 0x00, 0x00];
+
+ /**
+ * If the [rawBytes] have the valid version and checksum, extract and
+ * return the data from it. Otherwise return `null`.
+ */
+ List<int> getData(List<int> rawBytes) {
+ // There must be at least the version and the checksum in the raw bytes.
+ if (rawBytes.length < 8) {
+ return null;
+ }
+ int len = rawBytes.length - 8;
+
+ // Check the version.
+ if (rawBytes[len + 0] != _VERSION[0] ||
+ rawBytes[len + 1] != _VERSION[1] ||
+ rawBytes[len + 2] != _VERSION[2] ||
+ rawBytes[len + 3] != _VERSION[3]) {
+ return null;
+ }
+
+ // Check the CRC32 of the data.
+ List<int> data = rawBytes.sublist(0, len);
+ int crc = getCrc32(data);
+ if (rawBytes[len + 4] != crc & 0xFF ||
+ rawBytes[len + 5] != (crc >> 8) & 0xFF ||
+ rawBytes[len + 6] != (crc >> 16) & 0xFF ||
+ rawBytes[len + 7] != (crc >> 24) & 0xFF) {
+ return null;
+ }
+
+ // OK, the data is probably valid.
+ return data;
+ }
+
+ /**
+ * Return bytes that include the given [data] plus the current version and
+ * the checksum of the [data].
+ */
+ List<int> wrapData(List<int> data) {
+ int len = data.length;
+ var bytes = new Uint8List(len + 8);
+
+ // Put the data.
+ bytes.setRange(0, len, data);
+
+ // Put the version.
+ bytes[len + 0] = _VERSION[0];
+ bytes[len + 1] = _VERSION[1];
+ bytes[len + 2] = _VERSION[2];
+ bytes[len + 3] = _VERSION[3];
+
+ // Put the CRC32 of the data.
+ int crc = getCrc32(data);
+ bytes[len + 4] = crc & 0xFF;
+ bytes[len + 5] = (crc >> 8) & 0xFF;
+ bytes[len + 6] = (crc >> 16) & 0xFF;
+ bytes[len + 7] = (crc >> 24) & 0xFF;
+
+ return bytes;
+ }
+}
diff --git a/pkg/front_end/test/src/byte_store/crc32_test.dart b/pkg/front_end/test/src/byte_store/crc32_test.dart
new file mode 100644
index 0000000..0119bb8
--- /dev/null
+++ b/pkg/front_end/test/src/byte_store/crc32_test.dart
@@ -0,0 +1,38 @@
+// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:front_end/src/byte_store/crc32.dart';
+import 'package:test/test.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+main() {
+ defineReflectiveSuite(() {
+ defineReflectiveTests(Crc32Test);
+ });
+}
+
+@reflectiveTest
+class Crc32Test {
+ test_bytes_0() {
+ expect(getCrc32([]), 0x00000000);
+ }
+
+ test_bytes_1() {
+ expect(getCrc32([0x00]), 0xD202EF8D);
+ }
+
+ test_bytes_2() {
+ expect(getCrc32([0x01]), 0xA505DF1B);
+ }
+
+ test_bytes_3() {
+ expect(getCrc32([0x01, 0x02]), 0xB6CC4292);
+ }
+
+ test_string() {
+ expect(getCrc32(UTF8.encode('My very long test string.')), 0x88B8252E);
+ }
+}
diff --git a/pkg/front_end/test/src/byte_store/file_byte_store_test.dart b/pkg/front_end/test/src/byte_store/file_byte_store_test.dart
new file mode 100644
index 0000000..a2aa7ed
--- /dev/null
+++ b/pkg/front_end/test/src/byte_store/file_byte_store_test.dart
@@ -0,0 +1,64 @@
+// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:front_end/src/byte_store/file_byte_store.dart';
+import 'package:test/test.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+main() {
+ defineReflectiveSuite(() {
+ defineReflectiveTests(FileByteStoreValidatorTest);
+ });
+}
+
+@reflectiveTest
+class FileByteStoreValidatorTest {
+ final validator = new FileByteStoreValidator();
+
+ test_get_bad_notEnoughBytes() {
+ List<int> bytes = <int>[1, 2, 3];
+ List<int> data = validator.getData(bytes);
+ expect(data, isNull);
+ }
+
+ test_get_bad_notEnoughBytes_zero() {
+ List<int> bytes = <int>[];
+ List<int> data = validator.getData(bytes);
+ expect(data, isNull);
+ }
+
+ test_get_bad_wrongChecksum() {
+ List<int> data = <int>[1, 2, 3];
+ List<int> bytes = validator.wrapData(data);
+
+ // Damage the checksum.
+ expect(bytes[bytes.length - 1], isNot(42));
+ bytes[bytes.length - 1] = 42;
+
+ List<int> data2 = validator.getData(bytes);
+ expect(data2, isNull);
+ }
+
+ test_get_bad_wrongVersion() {
+ List<int> bytes = <int>[0xBA, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
+ List<int> data = validator.getData(bytes);
+ expect(data, isNull);
+ }
+
+ test_get_good() {
+ List<int> data = <int>[1, 2, 3];
+ List<int> bytes = validator.wrapData(data);
+ List<int> data2 = validator.getData(bytes);
+ expect(data2, hasLength(3));
+ expect(data2, data);
+ }
+
+ test_get_good_zeroBytesData() {
+ List<int> data = <int>[];
+ List<int> bytes = validator.wrapData(data);
+ List<int> data2 = validator.getData(bytes);
+ expect(data2, hasLength(0));
+ expect(data2, data);
+ }
+}
diff --git a/pkg/front_end/test/src/byte_store/protected_file_byte_store_test.dart b/pkg/front_end/test/src/byte_store/protected_file_byte_store_test.dart
index 6bb60d3..d117bf4 100644
--- a/pkg/front_end/test/src/byte_store/protected_file_byte_store_test.dart
+++ b/pkg/front_end/test/src/byte_store/protected_file_byte_store_test.dart
@@ -22,6 +22,8 @@
@reflectiveTest
class ProtectedFileByteStoreTest {
+ static const PADDING = 8;
+
io.Directory cacheDirectory;
String cachePath;
ProtectedFileByteStore store;
@@ -167,7 +169,7 @@
}
includes.forEach((expectedKey, expectedLength) {
expect(keyToLength, contains(expectedKey));
- expect(keyToLength, containsPair(expectedKey, expectedLength));
+ expect(keyToLength, containsPair(expectedKey, expectedLength + PADDING));
});
for (var excludedKey in excludes) {
expect(keyToLength.keys, isNot(contains(excludedKey)));
diff --git a/pkg/front_end/test/src/byte_store/test_all.dart b/pkg/front_end/test/src/byte_store/test_all.dart
index f702dcb..49fb3b4 100644
--- a/pkg/front_end/test/src/byte_store/test_all.dart
+++ b/pkg/front_end/test/src/byte_store/test_all.dart
@@ -6,6 +6,8 @@
import 'byte_store_test.dart' as byte_store_test;
import 'cache_test.dart' as cache_test;
+import 'crc32_test.dart' as crc32_test;
+import 'file_byte_store_test.dart' as file_byte_store_test;
import 'protected_file_byte_store_test.dart' as protected_file_byte_store_test;
/// Utility for manually running all tests.
@@ -13,6 +15,8 @@
defineReflectiveSuite(() {
byte_store_test.main();
cache_test.main();
+ crc32_test.main();
+ file_byte_store_test.main();
protected_file_byte_store_test.main();
}, name: 'byte_store');
}
