Add a string_scanner package. This code has been pulled out of the Shelf package. R=kevmoo@google.com Review URL: https://codereview.chromium.org//213833013 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart/pkg/string_scanner@34665 260f80e4-7a28-3924-810f-c04153c831b5
diff --git a/pkgs/string_scanner/LICENSE b/pkgs/string_scanner/LICENSE new file mode 100644 index 0000000..5c60afe --- /dev/null +++ b/pkgs/string_scanner/LICENSE
@@ -0,0 +1,26 @@ +Copyright 2014, the Dart project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/pkgs/string_scanner/README.md b/pkgs/string_scanner/README.md new file mode 100644 index 0000000..90660fc --- /dev/null +++ b/pkgs/string_scanner/README.md
@@ -0,0 +1,37 @@ +This package exposes a `StringScanner` type that makes it easy to parse a string +using a series of `Pattern`s. For example: + +```dart +import 'dart:math'; + +import 'package:string_scanner/string_scanner.dart'; + +num parseNumber(String source) { + // Scan a number ("1", "1.5", "-3"). + var scanner = new StringScanner(source); + + // [Scanner.scan] tries to consume a [Pattern] and returns whether or not it + // succeeded. It will move the scan pointer past the end of the pattern. + var negative = scanner.scan("-"); + + // [Scanner.expect] consumes a [Pattern] and throws a [FormatError] if it + // fails. Like [Scanner.scan], it will move the scan pointer forward. + scanner.expect(new RegExp(r"\d+")); + + // [Scanner.lastMatch] holds the [MatchData] for the most recent call to + // [Scanner.scan], [Scanner.expect], or [Scanner.matches]. + var number = int.parse(scanner.lastMatch[0]); + + if (scanner.scan(".")) { + scanner.expect(new RegExp(r"\d+")); + var decimal = scanner.lastMatch[0]; + number += int.parse(decimal) / math.pow(10, decimal.length); + } + + // [Scanner.expectDone] will throw a [FormatError] if there's any input that + // hasn't yet been consumed. + scanner.expectDone(); + + return (negative ? -1 : 1) * number; +} +```
diff --git a/pkgs/string_scanner/lib/string_scanner.dart b/pkgs/string_scanner/lib/string_scanner.dart new file mode 100644 index 0000000..624c090 --- /dev/null +++ b/pkgs/string_scanner/lib/string_scanner.dart
@@ -0,0 +1,113 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// A library for parsing strings using a sequence of patterns. +library string_scanner; + +// TODO(nweiz): Add some integration between this and source maps. +/// A class that scans through a string using [Pattern]s. +class StringScanner { + /// The string being scanned through. + final String string; + + /// The current position of the scanner in the string, in characters. + int get position => _position; + set position(int position) { + if (position < 0 || position > string.length) { + throw new ArgumentError("Invalid position $position"); + } + + _position = position; + } + int _position = 0; + + /// The data about the previous match made by the scanner. + /// + /// If the last match failed, this will be `null`. + Match get lastMatch => _lastMatch; + Match _lastMatch; + + /// The portion of the string that hasn't yet been scanned. + String get rest => string.substring(position); + + /// Whether the scanner has completely consumed [string]. + bool get isDone => position == string.length; + + /// Creates a new [StringScanner] that starts scanning from [position]. + /// + /// [position] defaults to 0, the beginning of the string. + StringScanner(this.string, {int position}) { + if (position != null) this.position = position; + } + + /// If [pattern] matches at the current position of the string, scans forward + /// until the end of the match. + /// + /// Returns whether or not [pattern] matched. + bool scan(Pattern pattern) { + var success = matches(pattern); + if (success) _position = _lastMatch.end; + return success; + } + + /// If [pattern] matches at the current position of the string, scans forward + /// until the end of the match. + /// + /// If [pattern] did not match, throws a [FormatException] describing the + /// position of the failure. [name] is used in this error as the expected name + /// of the pattern being matched; if it's `null`, the pattern itself is used + /// instead. + void expect(Pattern pattern, {String name}) { + if (scan(pattern)) return; + + if (name == null) { + if (pattern is RegExp) { + name = "/${pattern.pattern.replaceAll("/", "\\/")}/"; + } else { + name = pattern.toString() + .replaceAll("\\", "\\\\").replaceAll('"', '\\"'); + name = '"$name"'; + } + } + _fail(name); + } + + /// If the string has not been fully consumed, this throws a + /// [FormatException]. + void expectDone() { + if (isDone) return; + _fail("no more input"); + } + + /// Returns whether or not [pattern] matches at the current position of the + /// string. + /// + /// This doesn't move the scan pointer forward. + bool matches(Pattern pattern) { + _lastMatch = pattern.matchAsPrefix(string, position); + return _lastMatch != null; + } + + // TODO(nweiz): Make this handle long lines more gracefully. + /// Throws a [FormatException] describing that [name] is expected at the + /// current position in the string. + void _fail(String name) { + var newlines = "\n".allMatches(string.substring(0, position)).toList(); + var line = newlines.length + 1; + var column; + var lastLine; + if (newlines.isEmpty) { + column = position + 1; + lastLine = string.substring(0, position); + } else { + column = position - newlines.last.end + 1; + lastLine = string.substring(newlines.last.end, position); + } + lastLine += rest.replaceFirst(new RegExp(r"\n.*"), ''); + throw new FormatException( + "Expected $name on line $line, column $column.\n" + "$lastLine\n" + "${new List.filled(column - 1, ' ').join()}^"); + } +}
diff --git a/pkgs/string_scanner/pubspec.yaml b/pkgs/string_scanner/pubspec.yaml new file mode 100644 index 0000000..3035901 --- /dev/null +++ b/pkgs/string_scanner/pubspec.yaml
@@ -0,0 +1,10 @@ +name: string_scanner +version: 0.0.1 +author: "Dart Team <misc@dartlang.org>" +homepage: http://www.dartlang.org +description: > + A class for parsing strings using a sequence of patterns. +dev_dependencies: + unittest: ">=0.10.0 <0.11.0" +environment: + sdk: ">=1.2.0 <2.0.0"
diff --git a/pkgs/string_scanner/test/error_format_test.dart b/pkgs/string_scanner/test/error_format_test.dart new file mode 100644 index 0000000..1187344 --- /dev/null +++ b/pkgs/string_scanner/test/error_format_test.dart
@@ -0,0 +1,110 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library string_scanner.error_format_test; + +import 'package:string_scanner/string_scanner.dart'; +import 'package:unittest/unittest.dart'; + +void main() { + test('points to the first unconsumed character', () { + var scanner = new StringScanner('foo bar baz'); + scanner.expect('foo '); + expect(() => scanner.expect('foo'), throwsFormattedError(''' +Expected "foo" on line 1, column 5. +foo bar baz + ^''')); + }); + + test('prints the correct line', () { + var scanner = new StringScanner('foo bar baz\ndo re mi\nearth fire water'); + scanner.expect('foo bar baz\ndo '); + expect(() => scanner.expect('foo'), throwsFormattedError(''' +Expected "foo" on line 2, column 4. +do re mi + ^''')); + }); + + test('handles the beginning of the string correctly', () { + var scanner = new StringScanner('foo bar baz'); + expect(() => scanner.expect('zap'), throwsFormattedError(''' +Expected "zap" on line 1, column 1. +foo bar baz +^''')); + }); + + test('handles the end of the string correctly', () { + var scanner = new StringScanner('foo bar baz'); + scanner.expect('foo bar baz'); + expect(() => scanner.expect('bang'), throwsFormattedError(''' +Expected "bang" on line 1, column 12. +foo bar baz + ^''')); + }); + + test('handles an empty string correctly', () { + expect(() => new StringScanner('').expect('foo'), throwsFormattedError(''' +Expected "foo" on line 1, column 1. + +^''')); + }); + + group("expected name", () { + test("uses the provided name", () { + expect(() => new StringScanner('').expect('foo bar', name: 'zap'), + throwsFormattedError(''' +Expected zap on line 1, column 1. + +^''')); + }); + + test("escapes string quotes", () { + expect(() => new StringScanner('').expect('foo"bar'), + throwsFormattedError(''' +Expected "foo\\"bar" on line 1, column 1. + +^''')); + }); + + test("escapes string backslashes", () { + expect(() => new StringScanner('').expect('foo\\bar'), + throwsFormattedError(''' +Expected "foo\\\\bar" on line 1, column 1. + +^''')); + }); + + test("prints PERL-style regexps", () { + expect(() => new StringScanner('').expect(new RegExp(r'foo')), + throwsFormattedError(''' +Expected /foo/ on line 1, column 1. + +^''')); + }); + + test("escape regexp forward slashes", () { + expect(() => new StringScanner('').expect(new RegExp(r'foo/bar')), + throwsFormattedError(''' +Expected /foo\\/bar/ on line 1, column 1. + +^''')); + }); + + test("does not escape regexp backslashes", () { + expect(() => new StringScanner('').expect(new RegExp(r'foo\bar')), + throwsFormattedError(''' +Expected /foo\\bar/ on line 1, column 1. + +^''')); + }); + }); +} + +Matcher throwsFormattedError(String format) { + return throwsA(predicate((error) { + expect(error, isFormatException); + expect(error.message, equals(format)); + return true; + })); +}
diff --git a/pkgs/string_scanner/test/string_scanner_test.dart b/pkgs/string_scanner/test/string_scanner_test.dart new file mode 100644 index 0000000..0cab627 --- /dev/null +++ b/pkgs/string_scanner/test/string_scanner_test.dart
@@ -0,0 +1,267 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library string_scanner.string_scanner_test; + +import 'package:string_scanner/string_scanner.dart'; +import 'package:unittest/unittest.dart'; + +void main() { + group('with an empty string', () { + var scanner; + setUp(() { + scanner = new StringScanner(''); + }); + + test('is done', () { + expect(scanner.isDone, isTrue); + expect(scanner.expectDone, isNot(throwsFormatException)); + }); + + test('rest is empty', () { + expect(scanner.rest, isEmpty); + }); + + test('lastMatch is null', () { + expect(scanner.lastMatch, isNull); + }); + + test('position is zero', () { + expect(scanner.position, equals(0)); + }); + + test("scan returns false and doesn't change the state", () { + expect(scanner.scan(new RegExp('.')), isFalse); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(0)); + }); + + test("expect throws a FormatException and doesn't change the state", () { + expect(() => scanner.expect(new RegExp('.')), throwsFormatException); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(0)); + }); + + test("matches returns false and doesn't change the state", () { + expect(scanner.matches(new RegExp('.')), isFalse); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(0)); + }); + + test('setting position to 1 throws an ArgumentError', () { + expect(() { + scanner.position = 1; + }, throwsArgumentError); + }); + + test('setting position to -1 throws an ArgumentError', () { + expect(() { + scanner.position = -1; + }, throwsArgumentError); + }); + }); + + group('at the beginning of a string', () { + var scanner; + setUp(() { + scanner = new StringScanner('foo bar'); + }); + + test('is not done', () { + expect(scanner.isDone, isFalse); + expect(scanner.expectDone, throwsFormatException); + }); + + test('rest is the whole string', () { + expect(scanner.rest, equals('foo bar')); + }); + + test('lastMatch is null', () { + expect(scanner.lastMatch, isNull); + }); + + test('position is zero', () { + expect(scanner.position, equals(0)); + }); + + test("a matching scan returns true and changes the state", () { + expect(scanner.scan(new RegExp('f(..)')), isTrue); + expect(scanner.lastMatch[1], equals('oo')); + expect(scanner.position, equals(3)); + expect(scanner.rest, equals(' bar')); + }); + + test("a non-matching scan returns false and sets lastMatch to null", () { + expect(scanner.matches(new RegExp('f(..)')), isTrue); + expect(scanner.lastMatch, isNotNull); + + expect(scanner.scan(new RegExp('b(..)')), isFalse); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(0)); + expect(scanner.rest, equals('foo bar')); + }); + + test("a matching expect changes the state", () { + scanner.expect(new RegExp('f(..)')); + expect(scanner.lastMatch[1], equals('oo')); + expect(scanner.position, equals(3)); + expect(scanner.rest, equals(' bar')); + }); + + test("a non-matching expect throws a FormatException and sets lastMatch to " + "null", () { + expect(scanner.matches(new RegExp('f(..)')), isTrue); + expect(scanner.lastMatch, isNotNull); + + expect(() => scanner.expect(new RegExp('b(..)')), throwsFormatException); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(0)); + expect(scanner.rest, equals('foo bar')); + }); + + test("a matching matches returns true and only changes lastMatch", () { + expect(scanner.matches(new RegExp('f(..)')), isTrue); + expect(scanner.lastMatch[1], equals('oo')); + expect(scanner.position, equals(0)); + expect(scanner.rest, equals('foo bar')); + }); + + test("a non-matching matches returns false and doesn't change the state", + () { + expect(scanner.matches(new RegExp('b(..)')), isFalse); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(0)); + expect(scanner.rest, equals('foo bar')); + }); + + test('setting position to 1 moves the cursor forward', () { + scanner.position = 1; + expect(scanner.position, equals(1)); + expect(scanner.rest, equals('oo bar')); + + expect(scanner.scan(new RegExp('oo.')), isTrue); + expect(scanner.lastMatch[0], equals('oo ')); + expect(scanner.position, equals(4)); + expect(scanner.rest, equals('bar')); + }); + + test('setting position beyond the string throws an ArgumentError', () { + expect(() { + scanner.position = 8; + }, throwsArgumentError); + }); + + test('setting position to -1 throws an ArgumentError', () { + expect(() { + scanner.position = -1; + }, throwsArgumentError); + }); + + test('scan accepts any Pattern', () { + expect(scanner.scan('foo'), isTrue); + expect(scanner.lastMatch[0], equals('foo')); + expect(scanner.position, equals(3)); + expect(scanner.rest, equals(' bar')); + }); + + test('scans multiple times', () { + expect(scanner.scan(new RegExp('f(..)')), isTrue); + expect(scanner.lastMatch[1], equals('oo')); + expect(scanner.position, equals(3)); + expect(scanner.rest, equals(' bar')); + + expect(scanner.scan(new RegExp(' b(..)')), isTrue); + expect(scanner.lastMatch[1], equals('ar')); + expect(scanner.position, equals(7)); + expect(scanner.rest, equals('')); + expect(scanner.isDone, isTrue); + expect(scanner.expectDone, isNot(throwsFormatException)); + }); + }); + + group('at the end of a string', () { + var scanner; + setUp(() { + scanner = new StringScanner('foo bar'); + expect(scanner.scan('foo bar'), isTrue); + }); + + test('is done', () { + expect(scanner.isDone, isTrue); + expect(scanner.expectDone, isNot(throwsFormatException)); + }); + + test('rest is empty', () { + expect(scanner.rest, isEmpty); + }); + + test('position is zero', () { + expect(scanner.position, equals(7)); + }); + + test("scan returns false and sets lastMatch to null", () { + expect(scanner.scan(new RegExp('.')), isFalse); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(7)); + }); + + test("expect throws a FormatException and sets lastMatch to null", () { + expect(() => scanner.expect(new RegExp('.')), throwsFormatException); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(7)); + }); + + test("matches returns false sets lastMatch to null", () { + expect(scanner.matches(new RegExp('.')), isFalse); + expect(scanner.lastMatch, isNull); + expect(scanner.position, equals(7)); + }); + + test('setting position to 1 moves the cursor backward', () { + scanner.position = 1; + expect(scanner.position, equals(1)); + expect(scanner.rest, equals('oo bar')); + + expect(scanner.scan(new RegExp('oo.')), isTrue); + expect(scanner.lastMatch[0], equals('oo ')); + expect(scanner.position, equals(4)); + expect(scanner.rest, equals('bar')); + }); + + test('setting position beyond the string throws an ArgumentError', () { + expect(() { + scanner.position = 8; + }, throwsArgumentError); + }); + + test('setting position to -1 throws an ArgumentError', () { + expect(() { + scanner.position = -1; + }, throwsArgumentError); + }); + }); + + group('a scanner constructed with a custom position', () { + test('starts scanning from that position', () { + var scanner = new StringScanner('foo bar', position: 1); + expect(scanner.position, equals(1)); + expect(scanner.rest, equals('oo bar')); + + expect(scanner.scan(new RegExp('oo.')), isTrue); + expect(scanner.lastMatch[0], equals('oo ')); + expect(scanner.position, equals(4)); + expect(scanner.rest, equals('bar')); + }); + + test('throws an ArgumentError if the position is -1', () { + expect(() => new StringScanner('foo bar', position: -1), + throwsArgumentError); + }); + + test('throws an ArgumentError if the position is beyond the string', () { + expect(() => new StringScanner('foo bar', position: 8), + throwsArgumentError); + }); + }); +}