Fix fused UTF-8/JSON decoding.
The VM version's parser did not allow a leading BOM, and it failed to parse a top-level integer.
Added test to check this.
Fixes #33251
Bug: http://dartbug.com/33251
Change-Id: I51e429082f0e9baac81e20f73b0885922b40b0b8
Reviewed-on: https://dart-review.googlesource.com/56860
Commit-Queue: Lasse R.H. Nielsen <lrn@google.com>
Reviewed-by: Florian Loitsch <floitsch@google.com>
diff --git a/pkg/expect/lib/expect.dart b/pkg/expect/lib/expect.dart
index eea2ab4..4803de0 100644
--- a/pkg/expect/lib/expect.dart
+++ b/pkg/expect/lib/expect.dart
@@ -515,33 +515,35 @@
}
/**
- * Calls the function [f] and verifies that it throws an exception.
+ * Calls the function [f] and verifies that it throws a `T`.
* The optional [check] function can provide additional validation
- * that the correct exception is being thrown. For example, to check
- * the type of the exception you could write this:
+ * that the correct object is being thrown. For example, to check
+ * the content of the thrown boject you could write this:
*
- * Expect.throws(myThrowingFunction, (e) => e is MyException);
+ * Expect.throws<MyException>(myThrowingFunction,
+ * (e) => e.myMessage.contains("WARNING"));
+ *
+ * The type variable can be omitted and the type checked in [check]
+ * instead. This was traditionally done before Dart had generic methods.
*
* If `f` fails an expectation (i.e., throws an [ExpectException]), that
* exception is not caught by [Expect.throws]. The test is still considered
* failing.
*/
- static void throws(void f(), [bool check(Object error), String reason]) {
+ static void throws<T>(void f(), [bool check(T error), String reason]) {
String msg = reason == null ? "" : "($reason)";
- if (f is! _Nullary) {
+ if (f is! Function()) {
// Only throws from executing the function body should count as throwing.
// The failure to even call `f` should throw outside the try/catch.
_fail("Expect.throws$msg: Function f not callable with zero arguments");
}
try {
f();
- } catch (e, s) {
+ } on Object catch (e, s) {
// A test failure doesn't count as throwing.
if (e is ExpectException) rethrow;
- if (check != null && !check(e)) {
- _fail("Expect.throws$msg: Unexpected '$e'\n$s");
- }
- return;
+ if (e is T && (check == null || check(e))) return;
+ _fail("Expect.throws$msg: Unexpected '$e'\n$s");
}
_fail('Expect.throws$msg fails: Did not throw');
}
@@ -607,8 +609,9 @@
/// Used in [Expect] because [Expect.identical] shadows the real [identical].
bool _identical(a, b) => identical(a, b);
-typedef _Nullary(); // Expect.throws argument must be this type.
-
+/// Exception thrown on a failed expectation check.
+///
+/// Always recognized by [Expect.throws] as an unexpected error.
class ExpectException implements Exception {
final String message;
ExpectException(this.message);
diff --git a/runtime/lib/convert_patch.dart b/runtime/lib/convert_patch.dart
index b2e7ed2..a2c802d 100644
--- a/runtime/lib/convert_patch.dart
+++ b/runtime/lib/convert_patch.dart
@@ -63,6 +63,7 @@
parser.chunk = input;
parser.chunkEnd = input.length;
parser.parse(0);
+ parser.close();
return parser.result;
}
@@ -398,6 +399,7 @@
static const int KWD_NULL = 0; // Prefix of "null" seen.
static const int KWD_TRUE = 4; // Prefix of "true" seen.
static const int KWD_FALSE = 8; // Prefix of "false" seen.
+ static const int KWD_BOM = 12; // Prefix of BOM seen.
static const int KWD_COUNT_SHIFT = 4; // Prefix length in bits 4+.
// Mask used to mask off two lower bits.
@@ -439,8 +441,11 @@
* ..0ddd0011 : Partial 'null' keyword.
* ..0ddd0111 : Partial 'true' keyword.
* ..0ddd1011 : Partial 'false' keyword.
- * For all three keywords, the `ddd` bits encode the number
+ * ..0ddd1111 : Partial UTF-8 BOM byte seqeuence ("\xEF\xBB\xBF").
+ * For all keywords, the `ddd` bits encode the number
* of letters seen.
+ * The BOM byte sequence is only used by [_JsonUtf8Parser],
+ * and only at the very beginning of input.
*/
int partialState = NO_PARTIAL;
@@ -789,7 +794,8 @@
int keywordType = partialState & KWD_TYPE_MASK;
int count = partialState >> KWD_COUNT_SHIFT;
int keywordTypeIndex = keywordType >> KWD_TYPE_SHIFT;
- String keyword = const ["null", "true", "false"][keywordTypeIndex];
+ String keyword =
+ const ["null", "true", "false", "\xEF\xBB\xBF"][keywordTypeIndex];
assert(count < keyword.length);
do {
if (position == chunkEnd) {
@@ -798,13 +804,19 @@
return chunkEnd;
}
int expectedChar = keyword.codeUnitAt(count);
- if (getChar(position) != expectedChar) return fail(position);
+ if (getChar(position) != expectedChar) {
+ if (count == 0) {
+ assert(keywordType == KWD_BOM);
+ return position;
+ }
+ return fail(position);
+ }
position++;
count++;
} while (count < keyword.length);
if (keywordType == KWD_NULL) {
listener.handleNull();
- } else {
+ } else if (keywordType != KWD_BOM) {
listener.handleBool(keywordType == KWD_TRUE);
}
return position;
@@ -1255,7 +1267,7 @@
fail(position, "Missing expected digit");
} else {
// If it doesn't even start out as a numeral.
- fail(position, "Unexpected character");
+ fail(position);
}
}
if (digit == 0) {
@@ -1660,7 +1672,7 @@
position++;
} else {
throw new FormatException(
- "Unexepected UTF-8 continuation byte", utf8, position);
+ "Unexpected UTF-8 continuation byte", utf8, position);
}
} else if (char < 0xE0) {
// C0-DF
@@ -1726,7 +1738,11 @@
int chunkEnd;
_JsonUtf8Parser(_JsonListener listener, this.allowMalformed)
- : super(listener);
+ : super(listener) {
+ // Starts out checking for an optional BOM (KWD_BOM, count = 0).
+ partialState =
+ _ChunkedJsonParser.PARTIAL_KEYWORD | _ChunkedJsonParser.KWD_BOM;
+ }
int getChar(int position) => chunk[position];
diff --git a/tests/lib_2/convert/json_test.dart b/tests/lib_2/convert/json_test.dart
index d7be8a0..22e5f24 100644
--- a/tests/lib_2/convert/json_test.dart
+++ b/tests/lib_2/convert/json_test.dart
@@ -104,10 +104,16 @@
}
void testThrows(jsonText) {
+ var message = "json = '${escape(jsonText)}'";
Expect.throwsFormatException(() => json.decode(jsonText),
- "json = '${escape(jsonText)}'");
+ "json.decode, $message");
Expect.throwsFormatException(() => jsonDecode(jsonText),
- "json = '${escape(jsonText)}'");
+ "jsonDecode, $message");
+ Expect.throwsFormatException(() => json.decoder.convert(jsonText),
+ "json.decoder.convert, $message");
+ Expect.throwsFormatException(() =>
+ utf8.decoder.fuse(json.decoder).convert(utf8.encode(jsonText)),
+ "utf8.decoder.fuse(json.decoder) o utf.encode, $message");
}
testNumbers() {
@@ -187,6 +193,10 @@
testThrows("-2.2 e+2");
testThrows("-2.2e +2");
testThrows("-2.2e+ 2");
+ testThrows("01");
+ testThrows("0.");
+ testThrows(".0");
+ testThrows("0.e1");
testThrows("[2.,2]");
testThrows("{2.:2}");
diff --git a/tests/lib_2/convert/json_utf8_test.dart b/tests/lib_2/convert/json_utf8_test.dart
new file mode 100644
index 0000000..6afae4d
--- /dev/null
+++ b/tests/lib_2/convert/json_utf8_test.dart
@@ -0,0 +1,77 @@
+// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Test that the fused UTF-8/JSON decoder accepts a leading BOM.
+library test;
+
+import "package:expect/expect.dart";
+import "dart:convert";
+
+void main() {
+ for (var parse in [parseFuse, parseSequence, parseChunked]) {
+ // Sanity checks.
+ Expect.isTrue(parse('true'.codeUnits.toList()));
+ Expect.isFalse(parse('false'.codeUnits.toList()));
+ Expect.equals(123, parse('123'.codeUnits.toList()));
+ Expect.listEquals([42], parse('[42]'.codeUnits.toList()));
+ Expect.mapEquals({"x": 42}, parse('{"x":42}'.codeUnits.toList()));
+
+ // String (0x22 = ") with UTF-8 encoded Unicode characters.
+ Expect.equals(
+ "A\xff\u1234\u{65555}",
+ parse([
+ 0x22,
+ 0x41,
+ 0xc3,
+ 0xbf,
+ 0xe1,
+ 0x88,
+ 0xb4,
+ 0xf1,
+ 0xa5,
+ 0x95,
+ 0x95,
+ 0x22
+ ]));
+
+ // BOM followed by true.
+ Expect.isTrue(parse([0xEF, 0xBB, 0xBF, 0x74, 0x72, 0x75, 0x65]));
+ }
+
+ // Do not accept BOM in non-UTF-8 decoder.
+ Expect.throws<FormatException>(
+ () => new JsonDecoder().convert("\xEF\xBB\xBFtrue"));
+ Expect.throws<FormatException>(() => new JsonDecoder().convert("\uFEFFtrue"));
+
+ // Only accept BOM first.
+ Expect.throws<FormatException>(
+ () => parseFuse(" \xEF\xBB\xBFtrue".codeUnits.toList()));
+ // Only accept BOM first.
+ Expect.throws<FormatException>(
+ () => parseFuse(" true\xEF\xBB\xBF".codeUnits.toList()));
+
+ Expect.throws<FormatException>(
+ () => parseFuse(" [\xEF\xBB\xBF]".codeUnits.toList()));
+}
+
+Object parseFuse(List<int> text) {
+ return new Utf8Decoder().fuse(new JsonDecoder()).convert(text);
+}
+
+Object parseSequence(List<int> text) {
+ return new JsonDecoder().convert(new Utf8Decoder().convert(text));
+}
+
+Object parseChunked(List<int> text) {
+ var result;
+ var sink = new Utf8Decoder().fuse(new JsonDecoder()).startChunkedConversion(
+ new ChunkedConversionSink.withCallback((List<Object> values) {
+ result = values[0];
+ }));
+ for (var i = 0; i < text.length; i++) {
+ sink.add([text[i]]);
+ }
+ sink.close();
+ return result;
+}
