Rewrite the pkg/yaml parser. This increases the speed by about 10x. It also adds a number of new featurse. BUG=21317, 20859 R=rnystrom@google.com Review URL: https://codereview.chromium.org//689513002 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart/pkg/yaml@41649 260f80e4-7a28-3924-810f-c04153c831b5
diff --git a/pkgs/yaml/CHANGELOG.md b/pkgs/yaml/CHANGELOG.md index 68af5f5..f454255 100644 --- a/pkgs/yaml/CHANGELOG.md +++ b/pkgs/yaml/CHANGELOG.md
@@ -1,3 +1,21 @@ +## 2.1.0 + +* Rewrite the parser for a 10x speed improvement. + +* Support anchors and aliases (`&foo` and `*foo`). + +* Support explicit tags (e.g. `!!str`). Note that user-defined tags are still + not fully supported. + +* `%YAML` and `%TAG` directives are now parsed, although again user-defined tags + are not fully supported. + +* `YamlScalar`, `YamlList`, and `YamlMap` now expose the styles in which they + were written (for example plain vs folded, block vs flow). + +* A `yamlWarningCallback` field is exposed. This field can be used to customize + how YAML warnings are displayed. + ## 2.0.1+1 * Fix an import in a test.
diff --git a/pkgs/yaml/lib/src/composer.dart b/pkgs/yaml/lib/src/composer.dart deleted file mode 100644 index 8612067..0000000 --- a/pkgs/yaml/lib/src/composer.dart +++ /dev/null
@@ -1,183 +0,0 @@ -// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -library yaml.composer; - -import 'model.dart'; -import 'visitor.dart'; -import 'yaml_exception.dart'; - -/// Takes a parsed YAML document (what the spec calls the "serialization tree") -/// and resolves aliases, resolves tags, and parses scalars to produce the -/// "representation graph". -class Composer extends Visitor { - /// The root node of the serialization tree. - final Node _root; - - /// Map from anchor names to the most recent representation graph node with - /// that anchor. - final _anchors = <String, Node>{}; - - /// The next id to use for the represenation graph's anchors. - /// - /// The spec doesn't use anchors in the representation graph, but we do so - /// that the constructor can ensure that the same node in the representation - /// graph produces the same native object. - var _idCounter = 0; - - Composer(this._root); - - /// Runs the Composer to produce a representation graph. - Node compose() => _root.visit(this); - - /// Returns the anchor to which an alias node refers. - Node visitAlias(AliasNode alias) { - if (!_anchors.containsKey(alias.anchor)) { - throw new YamlException("No anchor for alias ${alias.anchor}.", - alias.span); - } - return _anchors[alias.anchor]; - } - - /// Parses a scalar node according to its tag, or auto-detects the type if no - /// tag exists. - /// - /// Currently this only supports the YAML core type schema. - Node visitScalar(ScalarNode scalar) { - if (scalar.tag.name == "!") { - return setAnchor(scalar, parseString(scalar)); - } else if (scalar.tag.name == "?") { - for (var fn in [parseNull, parseBool, parseInt, parseFloat]) { - var result = fn(scalar); - if (result != null) return result; - } - return setAnchor(scalar, parseString(scalar)); - } - - var result = _parseByTag(scalar); - if (result != null) return setAnchor(scalar, result); - throw new YamlException('Invalid literal for ${scalar.tag}.', - scalar.span); - } - - ScalarNode _parseByTag(ScalarNode scalar) { - switch (scalar.tag.name) { - case "null": return parseNull(scalar); - case "bool": return parseBool(scalar); - case "int": return parseInt(scalar); - case "float": return parseFloat(scalar); - case "str": return parseString(scalar); - } - throw new YamlException('Undefined tag: ${scalar.tag}.', scalar.span); - } - - /// Assigns a tag to the sequence and recursively composes its contents. - Node visitSequence(SequenceNode seq) { - var tagName = seq.tag.name; - if (tagName != "!" && tagName != "?" && tagName != Tag.yaml("seq")) { - throw new YamlException("Invalid tag for sequence: ${seq.tag}.", - seq.span); - } - - var result = setAnchor(seq, - new SequenceNode(Tag.yaml('seq'), null, seq.span)); - result.content = super.visitSequence(seq); - return result; - } - - /// Assigns a tag to the mapping and recursively composes its contents. - Node visitMapping(MappingNode map) { - var tagName = map.tag.name; - if (tagName != "!" && tagName != "?" && tagName != Tag.yaml("map")) { - throw new YamlException("Invalid tag for mapping: ${map.tag}.", - map.span); - } - - var result = setAnchor(map, - new MappingNode(Tag.yaml('map'), null, map.span)); - result.content = super.visitMapping(map); - return result; - } - - /// If the serialization tree node [anchored] has an anchor, records that - /// that anchor is pointing to the representation graph node [result]. - Node setAnchor(Node anchored, Node result) { - if (anchored.anchor == null) return result; - result.anchor = '${_idCounter++}'; - _anchors[anchored.anchor] = result; - return result; - } - - /// Parses a null scalar. - ScalarNode parseNull(ScalarNode scalar) { - if (new RegExp(r"^(null|Null|NULL|~|)$").hasMatch(scalar.content)) { - return new ScalarNode(Tag.yaml("null"), scalar.span, value: null); - } else { - return null; - } - } - - /// Parses a boolean scalar. - ScalarNode parseBool(ScalarNode scalar) { - var match = new RegExp(r"^(?:(true|True|TRUE)|(false|False|FALSE))$"). - firstMatch(scalar.content); - if (match == null) return null; - return new ScalarNode(Tag.yaml("bool"), scalar.span, - value: match.group(1) != null); - } - - /// Parses an integer scalar. - ScalarNode parseInt(ScalarNode scalar) { - var match = new RegExp(r"^[-+]?[0-9]+$").firstMatch(scalar.content); - if (match != null) { - return new ScalarNode(Tag.yaml("int"), scalar.span, - value: int.parse(match.group(0))); - } - - match = new RegExp(r"^0o([0-7]+)$").firstMatch(scalar.content); - if (match != null) { - int n = int.parse(match.group(1), radix: 8); - return new ScalarNode(Tag.yaml("int"), scalar.span, value: n); - } - - match = new RegExp(r"^0x[0-9a-fA-F]+$").firstMatch(scalar.content); - if (match != null) { - return new ScalarNode(Tag.yaml("int"), scalar.span, - value: int.parse(match.group(0))); - } - - return null; - } - - /// Parses a floating-point scalar. - ScalarNode parseFloat(ScalarNode scalar) { - var match = new RegExp( - r"^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$"). - firstMatch(scalar.content); - if (match != null) { - // YAML allows floats of the form "0.", but Dart does not. Fix up those - // floats by removing the trailing dot. - var matchStr = match.group(0).replaceAll(new RegExp(r"\.$"), ""); - return new ScalarNode(Tag.yaml("float"), scalar.span, - value: double.parse(matchStr)); - } - - match = new RegExp(r"^([+-]?)\.(inf|Inf|INF)$").firstMatch(scalar.content); - if (match != null) { - var value = match.group(1) == "-" ? -double.INFINITY : double.INFINITY; - return new ScalarNode(Tag.yaml("float"), scalar.span, value: value); - } - - match = new RegExp(r"^\.(nan|NaN|NAN)$").firstMatch(scalar.content); - if (match != null) { - return new ScalarNode(Tag.yaml("float"), scalar.span, value: double.NAN); - } - - return null; - } - - /// Parses a string scalar. - ScalarNode parseString(ScalarNode scalar) => - new ScalarNode(Tag.yaml("str"), scalar.span, value: scalar.content); -}
diff --git a/pkgs/yaml/lib/src/constructor.dart b/pkgs/yaml/lib/src/constructor.dart deleted file mode 100644 index 5931dae..0000000 --- a/pkgs/yaml/lib/src/constructor.dart +++ /dev/null
@@ -1,76 +0,0 @@ -// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -library yaml.constructor; - -import 'equality.dart'; -import 'model.dart'; -import 'visitor.dart'; -import 'yaml_node.dart'; - -/// Takes a parsed and composed YAML document (what the spec calls the -/// "representation graph") and creates native Dart objects that represent that -/// document. -class Constructor extends Visitor { - /// The root node of the representation graph. - final Node _root; - - /// Map from anchor names to the most recent Dart node with that anchor. - final _anchors = <String, YamlNode>{}; - - Constructor(this._root); - - /// Runs the Constructor to produce a Dart object. - YamlNode construct() => _root.visit(this); - - /// Returns the value of a scalar. - YamlScalar visitScalar(ScalarNode scalar) => - new YamlScalar.internal(scalar.value, scalar.span); - - /// Converts a sequence into a List of Dart objects. - YamlList visitSequence(SequenceNode seq) { - var anchor = getAnchor(seq); - if (anchor != null) return anchor; - var nodes = []; - var dartSeq = setAnchor(seq, new YamlList.internal(nodes, seq.span)); - nodes.addAll(super.visitSequence(seq)); - return dartSeq; - } - - /// Converts a mapping into a [Map] of Dart objects. - YamlMap visitMapping(MappingNode map) { - var anchor = getAnchor(map); - if (anchor != null) return anchor; - var nodes = deepEqualsMap(); - var dartMap = setAnchor(map, new YamlMap.internal(nodes, map.span)); - super.visitMapping(map).forEach((k, v) => nodes[k] = v); - return dartMap; - } - - /// Returns a new Dart object wrapping the object that already represents - /// [anchored], if such a thing exists. - YamlNode getAnchor(Node anchored) { - if (anchored.anchor == null) return null; - var value = _anchors[anchored.anchor]; - if (value == null) return null; - - // Re-wrap [value]'s contents so that it's associated with the span of the - // anchor rather than its original definition. - if (value is YamlMap) { - return new YamlMap.internal(value.nodes, anchored.span); - } else if (value is YamlList) { - return new YamlList.internal(value.nodes, anchored.span); - } else { - assert(value is YamlScalar); - return new YamlScalar.internal(value.value, anchored.span); - } - } - - /// Records that [value] is the Dart object representing [anchored]. - YamlNode setAnchor(Node anchored, YamlNode value) { - if (anchored.anchor == null) return value; - _anchors[anchored.anchor] = value; - return value; - } -}
diff --git a/pkgs/yaml/lib/src/event.dart b/pkgs/yaml/lib/src/event.dart new file mode 100644 index 0000000..96e2f16 --- /dev/null +++ b/pkgs/yaml/lib/src/event.dart
@@ -0,0 +1,157 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library yaml.event; + +import 'package:source_span/source_span.dart'; + +import 'style.dart'; +import 'yaml_document.dart'; + +/// An event emitted by a [Parser]. +class Event { + /// The event type. + final EventType type; + + /// The span associated with the event. + final FileSpan span; + + Event(this.type, this.span); + + String toString() => type.toString(); +} + +/// An event indicating the beginning of a YAML document. +class DocumentStartEvent implements Event { + get type => EventType.DOCUMENT_START; + final FileSpan span; + + /// The document's `%YAML` directive, or `null` if there was none. + final VersionDirective versionDirective; + + /// The document's `%TAG` directives, if any. + final List<TagDirective> tagDirectives; + + /// Whether the document started implicitly (that is, without an explicit + /// `===` sequence). + final bool isImplicit; + + DocumentStartEvent(this.span, {this.versionDirective, + List<TagDirective> tagDirectives, this.isImplicit: true}) + : tagDirectives = tagDirectives == null ? [] : tagDirectives; + + String toString() => "DOCUMENT_START"; +} + +/// An event indicating the end of a YAML document. +class DocumentEndEvent implements Event { + get type => EventType.DOCUMENT_END; + final FileSpan span; + + /// Whether the document ended implicitly (that is, without an explicit + /// `...` sequence). + final bool isImplicit; + + DocumentEndEvent(this.span, {this.isImplicit: true}); + + String toString() => "DOCUMENT_END"; +} + +/// An event indicating that an alias was referenced. +class AliasEvent implements Event { + get type => EventType.ALIAS; + final FileSpan span; + + /// The name of the anchor. + final String name; + + AliasEvent(this.span, this.name); + + String toString() => "ALIAS $name"; +} + +/// A base class for events that can have anchor and tag properties associated +/// with them. +abstract class _ValueEvent implements Event { + /// The name of the value's anchor, or `null` if it wasn't anchored. + String get anchor; + + /// The text of the value's tag, or `null` if it wasn't tagged. + String get tag; + + String toString() { + var buffer = new StringBuffer('$type'); + if (anchor != null) buffer.write(" &$anchor"); + if (tag != null) buffer.write(" $tag"); + return buffer.toString(); + } +} + +/// An event indicating a single scalar value. +class ScalarEvent extends _ValueEvent { + get type => EventType.SCALAR; + final FileSpan span; + final String anchor; + final String tag; + + /// The contents of the scalar. + final String value; + + /// The style of the scalar in the original source. + final ScalarStyle style; + + ScalarEvent(this.span, this.value, this.style, {this.anchor, this.tag}); + + String toString() => "${super.toString()} \"$value\""; +} + +/// An event indicating the beginning of a sequence. +class SequenceStartEvent extends _ValueEvent { + get type => EventType.SEQUENCE_START; + final FileSpan span; + final String anchor; + final String tag; + + /// The style of the collection in the original source. + final CollectionStyle style; + + SequenceStartEvent(this.span, this.style, {this.anchor, this.tag}); +} + +/// An event indicating the beginning of a mapping. +class MappingStartEvent extends _ValueEvent { + get type => EventType.MAPPING_START; + final FileSpan span; + final String anchor; + final String tag; + + /// The style of the collection in the original source. + final CollectionStyle style; + + MappingStartEvent(this.span, this.style, {this.anchor, this.tag}); +} + +/// An enum of types of [Event] object. +class EventType { + static const STREAM_START = const EventType._("STREAM_START"); + static const STREAM_END = const EventType._("STREAM_END"); + + static const DOCUMENT_START = const EventType._("DOCUMENT_START"); + static const DOCUMENT_END = const EventType._("DOCUMENT_END"); + + static const ALIAS = const EventType._("ALIAS"); + static const SCALAR = const EventType._("SCALAR"); + + static const SEQUENCE_START = const EventType._("SEQUENCE_START"); + static const SEQUENCE_END = const EventType._("SEQUENCE_END"); + + static const MAPPING_START = const EventType._("MAPPING_START"); + static const MAPPING_END = const EventType._("MAPPING_END"); + + final String name; + + const EventType._(this.name); + + String toString() => name; +}
diff --git a/pkgs/yaml/lib/src/loader.dart b/pkgs/yaml/lib/src/loader.dart new file mode 100644 index 0000000..d80578f --- /dev/null +++ b/pkgs/yaml/lib/src/loader.dart
@@ -0,0 +1,257 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library yaml.loader; + +import 'package:source_span/source_span.dart'; + +import 'equality.dart'; +import 'event.dart'; +import 'parser.dart'; +import 'yaml_document.dart'; +import 'yaml_exception.dart'; +import 'yaml_node.dart'; + +/// A loader that reads [Event]s emitted by a [Parser] and emits +/// [YamlDocument]s. +/// +/// This is based on the libyaml loader, available at +/// https://github.com/yaml/libyaml/blob/master/src/loader.c. The license for +/// that is available in ../../libyaml-license.txt. +class Loader { + /// The underlying [Parser] that generates [Event]s. + final Parser _parser; + + /// Aliases by the alias name. + final _aliases = new Map<String, YamlNode>(); + + /// The span of the entire stream emitted so far. + FileSpan get span => _span; + FileSpan _span; + + /// Creates a loader that loads [source]. + /// + /// [sourceUrl] can be a String or a [Uri]. + Loader(String source, {sourceUrl}) + : _parser = new Parser(source, sourceUrl: sourceUrl) { + var event = _parser.parse(); + _span = event.span; + assert(event.type == EventType.STREAM_START); + } + + /// Loads the next document from the stream. + /// + /// If there are no more documents, returns `null`. + YamlDocument load() { + if (_parser.isDone) return null; + + var event = _parser.parse(); + if (event.type == EventType.STREAM_END) { + _span = _span.expand(event.span); + return null; + } + + var document = _loadDocument(event); + _span = _span.expand(document.span); + _aliases.clear(); + return document; + } + + /// Composes a document object. + YamlDocument _loadDocument(DocumentStartEvent firstEvent) { + var contents = _loadNode(_parser.parse()); + + var lastEvent = _parser.parse(); + assert(lastEvent.type == EventType.DOCUMENT_END); + + return new YamlDocument.internal( + contents, + firstEvent.span.expand(lastEvent.span), + firstEvent.versionDirective, + firstEvent.tagDirectives, + startImplicit: firstEvent.isImplicit, + endImplicit: lastEvent.isImplicit); + } + + /// Composes a node. + YamlNode _loadNode(Event firstEvent) { + switch (firstEvent.type) { + case EventType.ALIAS: return _loadAlias(firstEvent); + case EventType.SCALAR: return _loadScalar(firstEvent); + case EventType.SEQUENCE_START: return _loadSequence(firstEvent); + case EventType.MAPPING_START: return _loadMapping(firstEvent); + default: throw "Unreachable"; + } + } + + /// Registers an anchor. + void _registerAnchor(String anchor, YamlNode node) { + if (anchor == null) return; + + // libyaml throws an error for duplicate anchors, but example 7.1 makes it + // clear that they should be overridden: + // http://yaml.org/spec/1.2/spec.html#id2786448. + + _aliases[anchor] = node; + } + + /// Composes a node corresponding to an alias. + YamlNode _loadAlias(AliasEvent event) { + var alias = _aliases[event.name]; + if (alias != null) return alias; + + throw new YamlException("Undefined alias.", event.span); + } + + /// Composes a scalar node. + YamlNode _loadScalar(ScalarEvent scalar) { + var node; + if (scalar.tag == "!") { + node = _parseString(scalar); + } else if (scalar.tag != null) { + node = _parseByTag(scalar); + } else { + node = _parseNull(scalar); + if (node == null) node = _parseBool(scalar); + if (node == null) node = _parseInt(scalar); + if (node == null) node = _parseFloat(scalar); + if (node == null) node = _parseString(scalar); + } + + _registerAnchor(scalar.anchor, node); + return node; + } + + /// Composes a sequence node. + YamlNode _loadSequence(SequenceStartEvent firstEvent) { + if (firstEvent.tag != "!" && firstEvent.tag != null && + firstEvent.tag != "tag:yaml.org,2002:seq") { + throw new YamlException("Invalid tag for sequence.", firstEvent.span); + } + + var children = []; + var node = new YamlList.internal( + children, firstEvent.span, firstEvent.style); + _registerAnchor(firstEvent.anchor, node); + + var event = _parser.parse(); + while (event.type != EventType.SEQUENCE_END) { + children.add(_loadNode(event)); + event = _parser.parse(); + } + + setSpan(node, firstEvent.span.expand(event.span)); + return node; + } + + /// Composes a mapping node. + YamlNode _loadMapping(MappingStartEvent firstEvent) { + if (firstEvent.tag != "!" && firstEvent.tag != null && + firstEvent.tag != "tag:yaml.org,2002:map") { + throw new YamlException("Invalid tag for mapping.", firstEvent.span); + } + + var children = deepEqualsMap(); + var node = new YamlMap.internal( + children, firstEvent.span, firstEvent.style); + _registerAnchor(firstEvent.anchor, node); + + var event = _parser.parse(); + while (event.type != EventType.MAPPING_END) { + var key = _loadNode(event); + var value = _loadNode(_parser.parse()); + children[key] = value; + event = _parser.parse(); + } + + setSpan(node, firstEvent.span.expand(event.span)); + return node; + } + + /// Parses a scalar according to its tag name. + YamlScalar _parseByTag(ScalarEvent scalar) { + switch (scalar.tag) { + case "tag:yaml.org,2002:null": return _parseNull(scalar); + case "tag:yaml.org,2002:bool": return _parseBool(scalar); + case "tag:yaml.org,2002:int": return _parseInt(scalar); + case "tag:yaml.org,2002:float": return _parseFloat(scalar); + case "tag:yaml.org,2002:str": return _parseString(scalar); + } + throw new YamlException('Undefined tag: ${scalar.tag}.', scalar.span); + } + + /// Parses a null scalar. + YamlScalar _parseNull(ScalarEvent scalar) { + // TODO(nweiz): stop using regexps. + // TODO(nweiz): add ScalarStyle and implicit metadata to the scalars. + if (new RegExp(r"^(null|Null|NULL|~|)$").hasMatch(scalar.value)) { + return new YamlScalar.internal(null, scalar.span, scalar.style); + } else { + return null; + } + } + + /// Parses a boolean scalar. + YamlScalar _parseBool(ScalarEvent scalar) { + var match = new RegExp(r"^(?:(true|True|TRUE)|(false|False|FALSE))$"). + firstMatch(scalar.value); + if (match == null) return null; + return new YamlScalar.internal( + match.group(1) != null, scalar.span, scalar.style); + } + + /// Parses an integer scalar. + YamlScalar _parseInt(ScalarEvent scalar) { + var match = new RegExp(r"^[-+]?[0-9]+$").firstMatch(scalar.value); + if (match != null) { + return new YamlScalar.internal( + int.parse(match.group(0)), scalar.span, scalar.style); + } + + match = new RegExp(r"^0o([0-7]+)$").firstMatch(scalar.value); + if (match != null) { + var n = int.parse(match.group(1), radix: 8); + return new YamlScalar.internal(n, scalar.span, scalar.style); + } + + match = new RegExp(r"^0x[0-9a-fA-F]+$").firstMatch(scalar.value); + if (match != null) { + return new YamlScalar.internal( + int.parse(match.group(0)), scalar.span, scalar.style); + } + + return null; + } + + /// Parses a floating-point scalar. + YamlScalar _parseFloat(ScalarEvent scalar) { + var match = new RegExp( + r"^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$"). + firstMatch(scalar.value); + if (match != null) { + // YAML allows floats of the form "0.", but Dart does not. Fix up those + // floats by removing the trailing dot. + var matchStr = match.group(0).replaceAll(new RegExp(r"\.$"), ""); + return new YamlScalar.internal( + double.parse(matchStr), scalar.span, scalar.style); + } + + match = new RegExp(r"^([+-]?)\.(inf|Inf|INF)$").firstMatch(scalar.value); + if (match != null) { + var value = match.group(1) == "-" ? -double.INFINITY : double.INFINITY; + return new YamlScalar.internal(value, scalar.span, scalar.style); + } + + match = new RegExp(r"^\.(nan|NaN|NAN)$").firstMatch(scalar.value); + if (match != null) { + return new YamlScalar.internal(double.NAN, scalar.span, scalar.style); + } + + return null; + } + + /// Parses a string scalar. + YamlScalar _parseString(ScalarEvent scalar) => + new YamlScalar.internal(scalar.value, scalar.span, scalar.style); +}
diff --git a/pkgs/yaml/lib/src/model.dart b/pkgs/yaml/lib/src/model.dart deleted file mode 100644 index 93cb49c..0000000 --- a/pkgs/yaml/lib/src/model.dart +++ /dev/null
@@ -1,259 +0,0 @@ -// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -/// This file contains the node classes for the internal representations of YAML -/// documents. These nodes are used for both the serialization tree and the -/// representation graph. -library yaml.model; - -import 'package:source_span/source_span.dart'; - -import 'equality.dart'; -import 'parser.dart'; -import 'visitor.dart'; -import 'yaml_exception.dart'; - -/// The prefix for tag types defined by the YAML spec. -const _YAML_URI_PREFIX = "tag:yaml.org,2002:"; - -/// A tag that indicates the type of a YAML node. -class Tag { - /// The name of the tag, either a URI or a local tag beginning with "!". - final String name; - - /// The kind of the tag. - final TagKind kind; - - /// Returns the standard YAML tag URI for [type]. - static String yaml(String type) => "tag:yaml.org,2002:$type"; - - const Tag(this.name, this.kind); - - const Tag.scalar(String name) - : this(name, TagKind.SCALAR); - - const Tag.sequence(String name) - : this(name, TagKind.SEQUENCE); - - const Tag.mapping(String name) - : this(name, TagKind.MAPPING); - - /// Two tags are equal if their URIs are equal. - operator ==(other) { - if (other is! Tag) return false; - return name == other.name; - } - - String toString() { - if (name.startsWith(_YAML_URI_PREFIX)) { - return '!!${name.substring(_YAML_URI_PREFIX.length)}'; - } else { - return '!<$name>'; - } - } - - int get hashCode => name.hashCode; -} - -/// An enum for kinds of tags. -class TagKind { - /// A tag indicating that the value is a scalar. - static const SCALAR = const TagKind._("scalar"); - - /// A tag indicating that the value is a sequence. - static const SEQUENCE = const TagKind._("sequence"); - - /// A tag indicating that the value is a mapping. - static const MAPPING = const TagKind._("mapping"); - - final String name; - - const TagKind._(this.name); - - String toString() => name; -} - -/// The abstract class for YAML nodes. -abstract class Node { - /// Every YAML node has a tag that describes its type. - Tag tag; - - /// Any YAML node can have an anchor associated with it. - String anchor; - - /// The source span for this node. - SourceSpan span; - - Node(this.tag, this.span, [this.anchor]); - - bool operator ==(other) { - if (other is! Node) return false; - return tag == other.tag; - } - - int get hashCode => tag.hashCode ^ anchor.hashCode; - - visit(Visitor v); -} - -/// A sequence node represents an ordered list of nodes. -class SequenceNode extends Node { - /// The nodes in the sequence. - List<Node> content; - - SequenceNode(String tagName, this.content, SourceSpan span) - : super(new Tag.sequence(tagName), span); - - /// Two sequences are equal if their tags and contents are equal. - bool operator ==(other) { - // Should be super != other; bug 2554 - if (!(super == other) || other is! SequenceNode) return false; - if (content.length != other.content.length) return false; - for (var i = 0; i < content.length; i++) { - if (content[i] != other.content[i]) return false; - } - return true; - } - - String toString() => '$tag [${content.map((e) => '$e').join(', ')}]'; - - int get hashCode => super.hashCode ^ deepHashCode(content); - - visit(Visitor v) => v.visitSequence(this); -} - -/// An alias node is a reference to an anchor. -class AliasNode extends Node { - AliasNode(String anchor, SourceSpan span) - : super(new Tag.scalar(Tag.yaml("str")), span, anchor); - - visit(Visitor v) => v.visitAlias(this); -} - -/// A scalar node represents all YAML nodes that have a single value. -class ScalarNode extends Node { - /// The string value of the scalar node, if it was created by the parser. - final String _content; - - /// The Dart value of the scalar node, if it was created by the composer. - final value; - - /// Creates a new Scalar node. - /// - /// Exactly one of [content] and [value] should be specified. Content should - /// be specified for a newly-parsed scalar that hasn't yet been composed. - /// Value should be specified for a composed scalar, although `null` is a - /// valid value. - ScalarNode(String tagName, SourceSpan span, {String content, this.value}) - : _content = content, - super(new Tag.scalar(tagName), span); - - /// Two scalars are equal if their string representations are equal. - bool operator ==(other) { - // Should be super != other; bug 2554 - if (!(super == other) || other is! ScalarNode) return false; - return content == other.content; - } - - /// Returns the string representation of the scalar. After composition, this - /// is equal to the canonical serialization of the value of the scalar. - String get content => _content != null ? _content : canonicalContent; - - /// Returns the canonical serialization of the value of the scalar. If the - /// value isn't given, the result of this will be "null". - String get canonicalContent { - if (value == null || value is bool || value is int) return '$value'; - - if (value is num) { - // 20 is the maximum value for this argument, which we use since YAML - // doesn't specify a maximum. - return value.toStringAsExponential(20). - replaceFirst(new RegExp("0+e"), "e"); - } - - if (value is String) { - // TODO(nweiz): This could be faster if we used a RegExp to check for - // special characters and short-circuited if they didn't exist. - - var escapedValue = value.codeUnits.map((c) { - switch (c) { - case Parser.TAB: return "\\t"; - case Parser.LF: return "\\n"; - case Parser.CR: return "\\r"; - case Parser.DOUBLE_QUOTE: return '\\"'; - case Parser.NULL: return "\\0"; - case Parser.BELL: return "\\a"; - case Parser.BACKSPACE: return "\\b"; - case Parser.VERTICAL_TAB: return "\\v"; - case Parser.FORM_FEED: return "\\f"; - case Parser.ESCAPE: return "\\e"; - case Parser.BACKSLASH: return "\\\\"; - case Parser.NEL: return "\\N"; - case Parser.NBSP: return "\\_"; - case Parser.LINE_SEPARATOR: return "\\L"; - case Parser.PARAGRAPH_SEPARATOR: return "\\P"; - default: - if (c < 0x20 || (c >= 0x7f && c < 0x100)) { - return "\\x${zeroPad(c.toRadixString(16).toUpperCase(), 2)}"; - } else if (c >= 0x100 && c < 0x10000) { - return "\\u${zeroPad(c.toRadixString(16).toUpperCase(), 4)}"; - } else if (c >= 0x10000) { - return "\\u${zeroPad(c.toRadixString(16).toUpperCase(), 8)}"; - } else { - return new String.fromCharCodes([c]); - } - } - }); - return '"${escapedValue.join()}"'; - } - - throw new YamlException('Unknown scalar value.', span); - } - - String toString() => '$tag "$content"'; - - /// Left-pads [str] with zeros so that it's at least [length] characters - /// long. - String zeroPad(String str, int length) { - assert(length >= str.length); - var prefix = new List.filled(length - str.length, '0'); - return '${prefix.join()}$str'; - } - - int get hashCode => super.hashCode ^ content.hashCode; - - visit(Visitor v) => v.visitScalar(this); -} - -/// A mapping node represents an unordered map of nodes to nodes. -class MappingNode extends Node { - /// The node map. - Map<Node, Node> content; - - MappingNode(String tagName, this.content, SourceSpan span) - : super(new Tag.mapping(tagName), span); - - /// Two mappings are equal if their tags and contents are equal. - bool operator ==(other) { - // Should be super != other; bug 2554 - if (!(super == other) || other is! MappingNode) return false; - if (content.length != other.content.length) return false; - for (var key in content.keys) { - if (!other.content.containsKey(key)) return false; - if (content[key] != other.content[key]) return false; - } - return true; - } - - String toString() { - var strContent = content.keys - .map((k) => '${k}: ${content[k]}') - .join(', '); - return '$tag {$strContent}'; - } - - int get hashCode => super.hashCode ^ deepHashCode(content); - - visit(Visitor v) => v.visitMapping(this); -}
diff --git a/pkgs/yaml/lib/src/parser.dart b/pkgs/yaml/lib/src/parser.dart index 94f551f..72c01dc 100644 --- a/pkgs/yaml/lib/src/parser.dart +++ b/pkgs/yaml/lib/src/parser.dart
@@ -1,1900 +1,817 @@ -// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. library yaml.parser; -import 'dart:collection'; - import 'package:source_span/source_span.dart'; import 'package:string_scanner/string_scanner.dart'; -import 'equality.dart'; -import 'model.dart'; +import 'event.dart'; +import 'scanner.dart'; +import 'style.dart'; +import 'token.dart'; import 'utils.dart'; +import 'yaml_document.dart'; +import 'yaml_exception.dart'; -/// Translates a string of characters into a YAML serialization tree. +/// A parser that reads [Token]s emitted by a [Scanner] and emits [Event]s. /// -/// This parser is designed to closely follow the spec. All productions in the -/// spec are numbered, and the corresponding methods in the parser have the same -/// numbers. This is certainly not the most efficient way of parsing YAML, but -/// it is the easiest to write and read in the context of the spec. -/// -/// Methods corresponding to productions are also named as in the spec, -/// translating the name of the method (although not the annotation characters) -/// into camel-case for dart style.. For example, the spec has a production -/// named `nb-ns-plain-in-line`, and the method implementing it is named -/// `nb_ns_plainInLine`. The exception to that rule is methods that just -/// recognize character classes; these are named `is*`. +/// This is based on the libyaml parser, available at +/// https://github.com/yaml/libyaml/blob/master/src/parser.c. The license for +/// that is available in ../../libyaml-license.txt. class Parser { - static const TAB = 0x9; - static const LF = 0xA; - static const CR = 0xD; - static const SP = 0x20; - static const TILDE = 0x7E; - static const NEL = 0x85; - static const PLUS = 0x2B; - static const HYPHEN = 0x2D; - static const QUESTION_MARK = 0x3F; - static const COLON = 0x3A; - static const COMMA = 0x2C; - static const LEFT_BRACKET = 0x5B; - static const RIGHT_BRACKET = 0x5D; - static const LEFT_BRACE = 0x7B; - static const RIGHT_BRACE = 0x7D; - static const HASH = 0x23; - static const AMPERSAND = 0x26; - static const ASTERISK = 0x2A; - static const EXCLAMATION = 0x21; - static const VERTICAL_BAR = 0x7C; - static const GREATER_THAN = 0x3E; - static const SINGLE_QUOTE = 0x27; - static const DOUBLE_QUOTE = 0x22; - static const PERCENT = 0x25; - static const AT = 0x40; - static const GRAVE_ACCENT = 0x60; + /// The underlying [Scanner] that generates [Token]s. + final Scanner _scanner; - static const NULL = 0x0; - static const BELL = 0x7; - static const BACKSPACE = 0x8; - static const VERTICAL_TAB = 0xB; - static const FORM_FEED = 0xC; - static const ESCAPE = 0x1B; - static const SLASH = 0x2F; - static const BACKSLASH = 0x5C; - static const UNDERSCORE = 0x5F; - static const NBSP = 0xA0; - static const LINE_SEPARATOR = 0x2028; - static const PARAGRAPH_SEPARATOR = 0x2029; + /// The stack of parse states for nested contexts. + final _states = new List<_State>(); - static const NUMBER_0 = 0x30; - static const NUMBER_9 = 0x39; + /// The current parse state. + var _state = _State.STREAM_START; - static const LETTER_A = 0x61; - static const LETTER_B = 0x62; - static const LETTER_E = 0x65; - static const LETTER_F = 0x66; - static const LETTER_N = 0x6E; - static const LETTER_R = 0x72; - static const LETTER_T = 0x74; - static const LETTER_U = 0x75; - static const LETTER_V = 0x76; - static const LETTER_X = 0x78; + /// The custom tag directives, by tag handle. + final _tagDirectives = new Map<String, TagDirective>(); - static const LETTER_CAP_A = 0x41; - static const LETTER_CAP_F = 0x46; - static const LETTER_CAP_L = 0x4C; - static const LETTER_CAP_N = 0x4E; - static const LETTER_CAP_P = 0x50; - static const LETTER_CAP_U = 0x55; - static const LETTER_CAP_X = 0x58; + /// Whether the parser has finished parsing. + bool get isDone => _state == _State.END; - static const C_SEQUENCE_ENTRY = 4; - static const C_MAPPING_KEY = 5; - static const C_MAPPING_VALUE = 6; - static const C_COLLECT_ENTRY = 7; - static const C_SEQUENCE_START = 8; - static const C_SEQUENCE_END = 9; - static const C_MAPPING_START = 10; - static const C_MAPPING_END = 11; - static const C_COMMENT = 12; - static const C_ANCHOR = 13; - static const C_ALIAS = 14; - static const C_TAG = 15; - static const C_LITERAL = 16; - static const C_FOLDED = 17; - static const C_SINGLE_QUOTE = 18; - static const C_DOUBLE_QUOTE = 19; - static const C_DIRECTIVE = 20; - static const C_RESERVED = 21; - - static const BLOCK_OUT = 0; - static const BLOCK_IN = 1; - static const FLOW_OUT = 2; - static const FLOW_IN = 3; - static const BLOCK_KEY = 4; - static const FLOW_KEY = 5; - - static const CHOMPING_STRIP = 0; - static const CHOMPING_KEEP = 1; - static const CHOMPING_CLIP = 2; - - /// The scanner that's used to scan through the document. - final SpanScanner _scanner; - - /// Whether we're parsing a bare document (that is, one that doesn't begin - /// with `---`). Bare documents don't allow `%` immediately following - /// newlines. - bool _inBareDocument = false; - - /// The state of the scanner when it was the farthest in the document it's - /// been. - LineScannerState _farthestState; - - /// The name of the context of the farthest position that has been parsed - /// successfully before backtracking. Used for error reporting. - String _farthestContext = "document"; - - /// A stack of the names of parse contexts. Used for error reporting. - final _contextStack = <String>["document"]; - - /// Annotations attached to ranges of the source string that add extra - /// information to any errors that occur in the annotated range. - final _errorAnnotations = new _RangeMap<String>(); - - /// The buffer containing the string currently being captured. - StringBuffer _capturedString; - - /// The beginning of the current section of the captured string. - int _captureStart; - - /// Whether the current string capture is being overridden. - bool _capturingAs = false; - - Parser(String yaml, sourceUrl) - : _scanner = new SpanScanner(yaml, sourceUrl: sourceUrl) { - _farthestState = _scanner.state; - } - - /// Returns the character at the current position, then moves that position - /// forward one character. - int next() => _scanner.readChar(); - - /// Returns the code unit at the current position, or the character [i] - /// characters after the current position. - int peek([int i = 0]) => _scanner.peekChar(i); - - /// The truthiness operator. Returns `false` if [obj] is `null` or `false`, - /// `true` otherwise. - bool truth(obj) => obj != null && obj != false; - - /// Consumes the current character if it matches [matcher]. Returns the result - /// of [matcher]. - bool consume(bool matcher(int)) { - if (matcher(peek())) { - next(); - return true; - } - return false; - } - - /// Consumes the current character if it equals [char]. - bool consumeChar(int char) => consume((c) => c == char); - - /// Calls [consumer] until it returns a falsey value. Returns a list of all - /// truthy return values of [consumer], or null if it didn't consume anything. + /// Creates a parser that parses [source]. /// - /// Conceptually, repeats a production one or more times. - List oneOrMore(consumer()) { - var first = consumer(); - if (!truth(first)) return null; - var out = [first]; - while (true) { - var el = consumer(); - if (!truth(el)) return out; - out.add(el); - } - return null; // Unreachable. - } + /// [sourceUrl] can be a String or a [Uri]. + Parser(String source, {sourceUrl}) + : _scanner = new Scanner(source, sourceUrl: sourceUrl); - /// Calls [consumer] until it returns a falsey value. Returns a list of all - /// truthy return values of [consumer], or the empty list if it didn't consume - /// anything. - /// - /// Conceptually, repeats a production any number of times. - List zeroOrMore(consumer()) { - var out = []; - var oldPos = _scanner.position; - while (true) { - var el = consumer(); - if (!truth(el) || oldPos == _scanner.position) return out; - oldPos = _scanner.position; - out.add(el); - } - return null; // Unreachable. - } - - /// Just calls [consumer] and returns its result. Used to make it explicit - /// that a production is intended to be optional. - zeroOrOne(consumer()) => consumer(); - - /// Calls each function in [consumers] until one returns a truthy value, then - /// returns that. - or(List<Function> consumers) { - for (var c in consumers) { - var res = c(); - if (truth(res)) return res; - } - return null; - } - - /// Calls [consumer] and returns its result, but rolls back the parser state - /// if [consumer] returns a falsey value. - transaction(consumer()) { - var oldState = _scanner.state; - var oldCaptureStart = _captureStart; - String capturedSoFar = _capturedString == null ? null : - _capturedString.toString(); - var res = consumer(); - _refreshFarthestState(); - if (truth(res)) return res; - - _scanner.state = oldState; - _captureStart = oldCaptureStart; - _capturedString = capturedSoFar == null ? null : - new StringBuffer(capturedSoFar); - return res; - } - - /// Consumes [n] characters matching [matcher], or none if there isn't a - /// complete match. The first argument to [matcher] is the character code, the - /// second is the index (from 0 to [n] - 1). - /// - /// Returns whether or not the characters were consumed. - bool nAtOnce(int n, bool matcher(int c, int i)) => transaction(() { - for (int i = 0; i < n; i++) { - if (!consume((c) => matcher(c, i))) return false; - } - return true; - }); - - /// Consumes the exact characters in [str], or nothing. - /// - /// Returns whether or not the string was consumed. - bool rawString(String str) => - nAtOnce(str.length, (c, i) => str.codeUnitAt(i) == c); - - /// Consumes and returns a string of characters matching [matcher], or null if - /// there are no such characters. - String stringOf(bool matcher(int)) => - captureString(() => oneOrMore(() => consume(matcher))); - - /// Calls [consumer] and returns the string that was consumed while doing so, - /// or null if [consumer] returned a falsey value. Automatically wraps - /// [consumer] in `transaction`. - String captureString(consumer()) { - // captureString calls may not be nested - assert(_capturedString == null); - - _captureStart = _scanner.position; - _capturedString = new StringBuffer(); - var res = transaction(consumer); - if (!truth(res)) { - _captureStart = null; - _capturedString = null; - return null; - } - - flushCapture(); - var result = _capturedString.toString(); - _captureStart = null; - _capturedString = null; - return result; - } - - captureAs(String replacement, consumer()) => - captureAndTransform(consumer, (_) => replacement); - - captureAndTransform(consumer(), String transformation(String captured)) { - if (_capturedString == null) return consumer(); - if (_capturingAs) return consumer(); - - flushCapture(); - _capturingAs = true; - var res = consumer(); - _capturingAs = false; - if (!truth(res)) return res; - - _capturedString.write(transformation( - _scanner.string.substring(_captureStart, _scanner.position))); - _captureStart = _scanner.position; - return res; - } - - void flushCapture() { - _capturedString.write(_scanner.string.substring( - _captureStart, _scanner.position)); - _captureStart = _scanner.position; - } - - /// Adds a tag and an anchor to [node], if they're defined. - Node addProps(Node node, Pair<Tag, String> props) { - if (props == null || node == null) return node; - if (truth(props.first)) node.tag = props.first; - if (truth(props.last)) node.anchor = props.last; - return node; - } - - /// Creates a MappingNode from [pairs]. - MappingNode map(List<Pair<Node, Node>> pairs, SourceSpan span) { - var content = new Map<Node, Node>(); - pairs.forEach((pair) => content[pair.first] = pair.last); - return new MappingNode("?", content, span); - } - - /// Runs [fn] in a context named [name]. Used for error reporting. - context(String name, fn()) { + /// Consumes and returns the next event. + Event parse() { try { - _contextStack.add(name); - return fn(); - } finally { - var popped = _contextStack.removeLast(); - assert(popped == name); + if (isDone) throw new StateError("No more events."); + var event = _stateMachine(); + return event; + } on StringScannerException catch (error) { + throw new YamlException(error.message, error.span); } } - /// Adds [message] as extra information to any errors that occur between the - /// current position and the position of the cursor after running [fn]. The - /// cursor is reset after [fn] is run. - annotateError(String message, fn()) { - var start = _scanner.position; - var end; - transaction(() { - fn(); - end = _scanner.position; - return false; - }); - _errorAnnotations[new _Range(start, end)] = message; + /// Dispatches parsing based on the current state. + Event _stateMachine() { + switch (_state) { + case _State.STREAM_START: + return _parseStreamStart(); + case _State.DOCUMENT_START: + return _parseDocumentStart(); + case _State.DOCUMENT_CONTENT: + return _parseDocumentContent(); + case _State.DOCUMENT_END: + return _parseDocumentEnd(); + case _State.BLOCK_NODE: + return _parseNode(block: true); + case _State.BLOCK_NODE_OR_INDENTLESS_SEQUENCE: + return _parseNode(block: true, indentlessSequence: true); + case _State.FLOW_NODE: + return _parseNode(); + case _State.BLOCK_SEQUENCE_FIRST_ENTRY: + // Scan past the `BLOCK-SEQUENCE-FIRST-ENTRY` token to the + // `BLOCK-SEQUENCE-ENTRY` token. + _scanner.scan(); + return _parseBlockSequenceEntry(); + case _State.BLOCK_SEQUENCE_ENTRY: + return _parseBlockSequenceEntry(); + case _State.INDENTLESS_SEQUENCE_ENTRY: + return _parseIndentlessSequenceEntry(); + case _State.BLOCK_MAPPING_FIRST_KEY: + // Scan past the `BLOCK-MAPPING-FIRST-KEY` token to the + // `BLOCK-MAPPING-KEY` token. + _scanner.scan(); + return _parseBlockMappingKey(); + case _State.BLOCK_MAPPING_KEY: + return _parseBlockMappingKey(); + case _State.BLOCK_MAPPING_VALUE: + return _parseBlockMappingValue(); + case _State.FLOW_SEQUENCE_FIRST_ENTRY: + return _parseFlowSequenceEntry(first: true); + case _State.FLOW_SEQUENCE_ENTRY: + return _parseFlowSequenceEntry(); + case _State.FLOW_SEQUENCE_ENTRY_MAPPING_KEY: + return _parseFlowSequenceEntryMappingKey(); + case _State.FLOW_SEQUENCE_ENTRY_MAPPING_VALUE: + return _parseFlowSequenceEntryMappingValue(); + case _State.FLOW_SEQUENCE_ENTRY_MAPPING_END: + return _parseFlowSequenceEntryMappingEnd(); + case _State.FLOW_MAPPING_FIRST_KEY: + return _parseFlowMappingKey(first: true); + case _State.FLOW_MAPPING_KEY: + return _parseFlowMappingKey(); + case _State.FLOW_MAPPING_VALUE: + return _parseFlowMappingValue(); + case _State.FLOW_MAPPING_EMPTY_VALUE: + return _parseFlowMappingValue(empty: true); + default: + throw "Unreachable"; + } } - /// Throws an error with additional context information. - void error(String message) => - _scanner.error("$message (in $_farthestContext)."); - - /// If [result] is falsey, throws an error saying that [expected] was - /// expected. - expect(result, String expected) { - if (truth(result)) return result; - error("Expected $expected"); - } - - /// Throws an error saying that the parse failed. + /// Parses the production: /// - /// Uses [_farthestState] and [_farthestContext] to provide additional - /// information. - parseFailed() { - var message = "Invalid YAML in $_farthestContext"; - _refreshFarthestState(); - _scanner.state = _farthestState; + /// stream ::= + /// STREAM-START implicit_document? explicit_document* STREAM-END + /// ************ + Event _parseStreamStart() { + var token = _scanner.scan(); + assert(token.type == TokenType.STREAM_START); - var extraError = _errorAnnotations[_scanner.position]; - if (extraError != null) message = "$message ($extraError)"; - _scanner.error("$message."); + _state = _State.DOCUMENT_START; + return new Event(EventType.STREAM_START, token.span); } - /// Update [_farthestState] if the scanner is farther than it's been before. - void _refreshFarthestState() { - if (_scanner.position <= _farthestState.position) return; - _farthestState = _scanner.state; - } + /// Parses the productions: + /// + /// implicit_document ::= block_node DOCUMENT-END* + /// * + /// explicit_document ::= + /// DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + /// ************************* + Event _parseDocumentStart() { + var token = _scanner.peek(); - /// Returns the number of spaces after the current position. - int countIndentation() { - var i = 0; - while (peek(i) == SP) i++; - return i; - } + // libyaml requires any document beyond the first in the stream to have an + // explicit document start indicator, but the spec allows it to be omitted + // as long as there was an end indicator. - /// Returns the indentation for a block scalar. - int blockScalarAdditionalIndentation(_BlockHeader header, int indent) { - if (!header.autoDetectIndent) return header.additionalIndent; - - var maxSpaces = 0; - var spaces = 0; - transaction(() { - do { - spaces = captureString(() => zeroOrMore(() => consumeChar(SP))).length; - if (spaces > maxSpaces) maxSpaces = spaces; - } while (b_break()); - return false; - }); - - // If the next non-empty line isn't indented further than the start of the - // block scalar, that means the scalar is going to be empty. Returning any - // value > 0 will cause the parser not to consume any text. - if (spaces <= indent) return 1; - - // It's an error for a leading empty line to be indented more than the first - // non-empty line. - if (maxSpaces > spaces) { - _scanner.error("Leading empty lines may not be indented more than the " - "first non-empty line."); + // Parse extra document end indicators. + while (token.type == TokenType.DOCUMENT_END) { + token = _scanner.advance(); } - return spaces - indent; + if (token.type != TokenType.VERSION_DIRECTIVE && + token.type != TokenType.TAG_DIRECTIVE && + token.type != TokenType.DOCUMENT_START && + token.type != TokenType.STREAM_END) { + // Parse an implicit document. + _processDirectives(); + _states.add(_State.DOCUMENT_END); + _state = _State.BLOCK_NODE; + return new DocumentStartEvent(token.span.start.pointSpan()); + } + + if (token.type == TokenType.STREAM_END) { + _state = _State.END; + _scanner.scan(); + return new Event(EventType.STREAM_END, token.span); + } + + // Parse an explicit document. + var start = token.span; + var pair = _processDirectives(); + var versionDirective = pair.first; + var tagDirectives = pair.last; + token = _scanner.peek(); + if (token.type != TokenType.DOCUMENT_START) { + throw new YamlException("Expected document start.", token.span); + } + + _states.add(_State.DOCUMENT_END); + _state = _State.DOCUMENT_CONTENT; + _scanner.scan(); + return new DocumentStartEvent(start.expand(token.span), + versionDirective: versionDirective, + tagDirectives: tagDirectives, + isImplicit: false); } - /// Returns whether the current position is at the beginning of a line. - bool get atStartOfLine => _scanner.column == 0; + /// Parses the productions: + /// + /// explicit_document ::= + /// DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + /// *********** + Event _parseDocumentContent() { + var token = _scanner.peek(); - /// Given an indicator character, returns the type of that indicator (or null - /// if the indicator isn't found. - int indicatorType(int char) { - switch (char) { - case HYPHEN: return C_SEQUENCE_ENTRY; - case QUESTION_MARK: return C_MAPPING_KEY; - case COLON: return C_MAPPING_VALUE; - case COMMA: return C_COLLECT_ENTRY; - case LEFT_BRACKET: return C_SEQUENCE_START; - case RIGHT_BRACKET: return C_SEQUENCE_END; - case LEFT_BRACE: return C_MAPPING_START; - case RIGHT_BRACE: return C_MAPPING_END; - case HASH: return C_COMMENT; - case AMPERSAND: return C_ANCHOR; - case ASTERISK: return C_ALIAS; - case EXCLAMATION: return C_TAG; - case VERTICAL_BAR: return C_LITERAL; - case GREATER_THAN: return C_FOLDED; - case SINGLE_QUOTE: return C_SINGLE_QUOTE; - case DOUBLE_QUOTE: return C_DOUBLE_QUOTE; - case PERCENT: return C_DIRECTIVE; - case AT: - case GRAVE_ACCENT: - return C_RESERVED; - default: return null; + switch (token.type) { + case TokenType.VERSION_DIRECTIVE: + case TokenType.TAG_DIRECTIVE: + case TokenType.DOCUMENT_START: + case TokenType.DOCUMENT_END: + case TokenType.STREAM_END: + _state = _states.removeLast(); + return _processEmptyScalar(token.span.start); + default: + return _parseNode(block: true); } } - // 1 - bool isPrintable(int char) { - if (char == null) return false; - return char == TAB || - char == LF || - char == CR || - (char >= SP && char <= TILDE) || - char == NEL || - (char >= 0xA0 && char <= 0xD7FF) || - (char >= 0xE000 && char <= 0xFFFD) || - (char >= 0x10000 && char <= 0x10FFFF); - } + /// Parses the productions: + /// + /// implicit_document ::= block_node DOCUMENT-END* + /// ************* + /// explicit_document ::= + /// DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + /// ************* + Event _parseDocumentEnd() { + _tagDirectives.clear(); + _state = _State.DOCUMENT_START; - // 2 - bool isJson(int char) => char != null && - (char == TAB || (char >= SP && char <= 0x10FFFF)); - - // 22 - bool c_indicator(int type) => consume((c) => indicatorType(c) == type); - - // 23 - bool isFlowIndicator(int char) { - var indicator = indicatorType(char); - return indicator == C_COLLECT_ENTRY || - indicator == C_SEQUENCE_START || - indicator == C_SEQUENCE_END || - indicator == C_MAPPING_START || - indicator == C_MAPPING_END; - } - - // 26 - bool isBreak(int char) => char == LF || char == CR; - - // 27 - bool isNonBreak(int char) => isPrintable(char) && !isBreak(char); - - // 28 - bool b_break() { - if (consumeChar(CR)) { - zeroOrOne(() => consumeChar(LF)); - return true; - } - return consumeChar(LF); - } - - // 29 - bool b_asLineFeed() => captureAs("\n", () => b_break()); - - // 30 - bool b_nonContent() => captureAs("", () => b_break()); - - // 33 - bool isSpace(int char) => char == SP || char == TAB; - - // 34 - bool isNonSpace(int char) => isNonBreak(char) && !isSpace(char); - - // 35 - bool isDecDigit(int char) => char != null && char >= NUMBER_0 && - char <= NUMBER_9; - - // 36 - bool isHexDigit(int char) { - if (char == null) return false; - return isDecDigit(char) || - (char >= LETTER_A && char <= LETTER_F) || - (char >= LETTER_CAP_A && char <= LETTER_CAP_F); - } - - // 41 - bool c_escape() => captureAs("", () => consumeChar(BACKSLASH)); - - // 42 - bool ns_escNull() => captureAs("\x00", () => consumeChar(NUMBER_0)); - - // 43 - bool ns_escBell() => captureAs("\x07", () => consumeChar(LETTER_A)); - - // 44 - bool ns_escBackspace() => captureAs("\b", () => consumeChar(LETTER_B)); - - // 45 - bool ns_escHorizontalTab() => captureAs("\t", () { - return consume((c) => c == LETTER_T || c == TAB); - }); - - // 46 - bool ns_escLineFeed() => captureAs("\n", () => consumeChar(LETTER_N)); - - // 47 - bool ns_escVerticalTab() => captureAs("\v", () => consumeChar(LETTER_V)); - - // 48 - bool ns_escFormFeed() => captureAs("\f", () => consumeChar(LETTER_F)); - - // 49 - bool ns_escCarriageReturn() => captureAs("\r", () => consumeChar(LETTER_R)); - - // 50 - bool ns_escEscape() => captureAs("\x1B", () => consumeChar(LETTER_E)); - - // 51 - bool ns_escSpace() => consumeChar(SP); - - // 52 - bool ns_escDoubleQuote() => consumeChar(DOUBLE_QUOTE); - - // 53 - bool ns_escSlash() => consumeChar(SLASH); - - // 54 - bool ns_escBackslash() => consumeChar(BACKSLASH); - - // 55 - bool ns_escNextLine() => captureAs("\x85", () => consumeChar(LETTER_CAP_N)); - - // 56 - bool ns_escNonBreakingSpace() => - captureAs("\xA0", () => consumeChar(UNDERSCORE)); - - // 57 - bool ns_escLineSeparator() => - captureAs("\u2028", () => consumeChar(LETTER_CAP_L)); - - // 58 - bool ns_escParagraphSeparator() => - captureAs("\u2029", () => consumeChar(LETTER_CAP_P)); - - // 59 - bool ns_esc8Bit() => ns_escNBit(LETTER_X, 2); - - // 60 - bool ns_esc16Bit() => ns_escNBit(LETTER_U, 4); - - // 61 - bool ns_esc32Bit() => ns_escNBit(LETTER_CAP_U, 8); - - // Helper method for 59 - 61 - bool ns_escNBit(int char, int digits) { - if (!captureAs('', () => consumeChar(char))) return false; - var captured = captureAndTransform( - () => nAtOnce(digits, (c, _) => isHexDigit(c)), - (hex) => new String.fromCharCodes([int.parse("0x$hex")])); - return expect(captured, "$digits hexidecimal digits"); - } - - // 62 - bool c_ns_escChar() => context('escape sequence', () => transaction(() { - if (!truth(c_escape())) return false; - return truth(or([ - ns_escNull, ns_escBell, ns_escBackspace, ns_escHorizontalTab, - ns_escLineFeed, ns_escVerticalTab, ns_escFormFeed, ns_escCarriageReturn, - ns_escEscape, ns_escSpace, ns_escDoubleQuote, ns_escSlash, - ns_escBackslash, ns_escNextLine, ns_escNonBreakingSpace, - ns_escLineSeparator, ns_escParagraphSeparator, ns_esc8Bit, ns_esc16Bit, - ns_esc32Bit - ])); - })); - - // 63 - bool s_indent(int indent) { - var result = nAtOnce(indent, (c, i) => c == SP); - if (peek() == TAB) { - annotateError("tab characters are not allowed as indentation in YAML", - () => zeroOrMore(() => consume(isSpace))); - } - return result; - } - - // 64 - bool s_indentLessThan(int indent) { - for (int i = 0; i < indent - 1; i++) { - if (!consumeChar(SP)) { - if (peek() == TAB) { - annotateError("tab characters are not allowed as indentation in YAML", - () { - for (; i < indent - 1; i++) { - if (!consume(isSpace)) break; - } - }); - } - break; - } - } - return true; - } - - // 65 - bool s_indentLessThanOrEqualTo(int indent) => s_indentLessThan(indent + 1); - - // 66 - bool s_separateInLine() => transaction(() { - return captureAs('', () => - truth(oneOrMore(() => consume(isSpace))) || atStartOfLine); - }); - - // 67 - bool s_linePrefix(int indent, int ctx) => captureAs("", () { - switch (ctx) { - case BLOCK_OUT: - case BLOCK_IN: - return s_blockLinePrefix(indent); - case FLOW_OUT: - case FLOW_IN: - return s_flowLinePrefix(indent); - } - }); - - // 68 - bool s_blockLinePrefix(int indent) => s_indent(indent); - - // 69 - bool s_flowLinePrefix(int indent) => captureAs('', () { - if (!truth(s_indent(indent))) return false; - zeroOrOne(s_separateInLine); - return true; - }); - - // 70 - bool l_empty(int indent, int ctx) => transaction(() { - var start = or([ - () => s_linePrefix(indent, ctx), - () => s_indentLessThan(indent) - ]); - if (!truth(start)) return false; - return b_asLineFeed(); - }); - - // 71 - bool b_asSpace() => captureAs(" ", () => consume(isBreak)); - - // 72 - bool b_l_trimmed(int indent, int ctx) => transaction(() { - if (!truth(b_nonContent())) return false; - return truth(oneOrMore(() => captureAs("\n", () => l_empty(indent, ctx)))); - }); - - // 73 - bool b_l_folded(int indent, int ctx) => - or([() => b_l_trimmed(indent, ctx), b_asSpace]); - - // 74 - bool s_flowFolded(int indent) => transaction(() { - zeroOrOne(s_separateInLine); - if (!truth(b_l_folded(indent, FLOW_IN))) return false; - return s_flowLinePrefix(indent); - }); - - // 75 - bool c_nb_commentText() { - if (!truth(c_indicator(C_COMMENT))) return false; - zeroOrMore(() => consume(isNonBreak)); - return true; - } - - // 76 - bool b_comment() => _scanner.isDone || b_nonContent(); - - // 77 - bool s_b_comment() { - if (truth(s_separateInLine())) { - zeroOrOne(c_nb_commentText); - } - return b_comment(); - } - - // 78 - bool l_comment() => transaction(() { - if (!truth(s_separateInLine())) return false; - zeroOrOne(c_nb_commentText); - return b_comment(); - }); - - // 79 - bool s_l_comments() { - if (!truth(s_b_comment()) && !atStartOfLine) return false; - zeroOrMore(l_comment); - return true; - } - - // 80 - bool s_separate(int indent, int ctx) { - switch (ctx) { - case BLOCK_OUT: - case BLOCK_IN: - case FLOW_OUT: - case FLOW_IN: - return s_separateLines(indent); - case BLOCK_KEY: - case FLOW_KEY: - return s_separateInLine(); - default: throw 'Invalid context "$ctx".'; + var token = _scanner.peek(); + if (token.type == TokenType.DOCUMENT_END) { + _scanner.scan(); + return new DocumentEndEvent(token.span, isImplicit: false); + } else { + return new DocumentEndEvent( + token.span.start.pointSpan(), isImplicit: true); } } - // 81 - bool s_separateLines(int indent) { - return transaction(() => s_l_comments() && s_flowLinePrefix(indent)) || - s_separateInLine(); - } + /// Parses the productions: + /// + /// block_node_or_indentless_sequence ::= + /// ALIAS + /// ***** + /// | properties (block_content | indentless_block_sequence)? + /// ********** * + /// | block_content | indentless_block_sequence + /// * + /// block_node ::= ALIAS + /// ***** + /// | properties block_content? + /// ********** * + /// | block_content + /// * + /// flow_node ::= ALIAS + /// ***** + /// | properties flow_content? + /// ********** * + /// | flow_content + /// * + /// properties ::= TAG ANCHOR? | ANCHOR TAG? + /// ************************* + /// block_content ::= block_collection | flow_collection | SCALAR + /// ****** + /// flow_content ::= flow_collection | SCALAR + /// ****** + Event _parseNode({bool block: false, bool indentlessSequence: false}) { + var token = _scanner.peek(); - // 82 - bool l_directive() => false; // TODO(nweiz): implement - - // 96 - Pair<Tag, String> c_ns_properties(int indent, int ctx) { - var tag, anchor; - tag = c_ns_tagProperty(); - if (truth(tag)) { - anchor = transaction(() { - if (!truth(s_separate(indent, ctx))) return null; - return c_ns_anchorProperty(); - }); - return new Pair<Tag, String>(tag, anchor); + if (token is AliasToken) { + _scanner.scan(); + _state = _states.removeLast(); + return new AliasEvent(token.span, token.name); } - anchor = c_ns_anchorProperty(); - if (truth(anchor)) { - tag = transaction(() { - if (!truth(s_separate(indent, ctx))) return null; - return c_ns_tagProperty(); - }); - return new Pair<Tag, String>(tag, anchor); + var anchor; + var tagToken; + var span = token.span.start.pointSpan(); + parseAnchor() { + anchor = token.name; + span = span.expand(token.span); + token = _scanner.advance(); } - return null; - } - - // 97 - Tag c_ns_tagProperty() => null; // TODO(nweiz): implement - - // 101 - String c_ns_anchorProperty() => null; // TODO(nweiz): implement - - // 102 - bool isAnchorChar(int char) => isNonSpace(char) && !isFlowIndicator(char); - - // 103 - String ns_anchorName() => - captureString(() => oneOrMore(() => consume(isAnchorChar))); - - // 104 - Node c_ns_aliasNode() { - var start = _scanner.state; - if (!truth(c_indicator(C_ALIAS))) return null; - var name = expect(ns_anchorName(), 'anchor name'); - return new AliasNode(name, _scanner.spanFrom(start)); - } - - // 105 - ScalarNode e_scalar() => new ScalarNode("?", _scanner.emptySpan, content: ""); - - // 106 - ScalarNode e_node() => e_scalar(); - - // 107 - bool nb_doubleChar() => or([ - c_ns_escChar, - () => consume((c) => isJson(c) && c != BACKSLASH && c != DOUBLE_QUOTE) - ]); - - // 108 - bool ns_doubleChar() => !isSpace(peek()) && truth(nb_doubleChar()); - - // 109 - Node c_doubleQuoted(int indent, int ctx) => context('string', () { - return transaction(() { - var start = _scanner.state; - if (!truth(c_indicator(C_DOUBLE_QUOTE))) return null; - var contents = nb_doubleText(indent, ctx); - if (!truth(c_indicator(C_DOUBLE_QUOTE))) return null; - return new ScalarNode("!", _scanner.spanFrom(start), content: contents); - }); - }); - - // 110 - String nb_doubleText(int indent, int ctx) => captureString(() { - switch (ctx) { - case FLOW_OUT: - case FLOW_IN: - nb_doubleMultiLine(indent); - break; - case BLOCK_KEY: - case FLOW_KEY: - nb_doubleOneLine(); - break; - } - return true; - }); - - // 111 - void nb_doubleOneLine() { - zeroOrMore(nb_doubleChar); - } - - // 112 - bool s_doubleEscaped(int indent) => transaction(() { - zeroOrMore(() => consume(isSpace)); - if (!captureAs("", () => consumeChar(BACKSLASH))) return false; - if (!truth(b_nonContent())) return false; - zeroOrMore(() => captureAs("\n", () => l_empty(indent, FLOW_IN))); - return s_flowLinePrefix(indent); - }); - - // 113 - bool s_doubleBreak(int indent) => or([ - () => s_doubleEscaped(indent), - () => s_flowFolded(indent) - ]); - - // 114 - void nb_ns_doubleInLine() { - zeroOrMore(() => transaction(() { - zeroOrMore(() => consume(isSpace)); - return ns_doubleChar(); - })); - } - - // 115 - bool s_doubleNextLine(int indent) { - if (!truth(s_doubleBreak(indent))) return false; - zeroOrOne(() { - if (!truth(ns_doubleChar())) return; - nb_ns_doubleInLine(); - or([ - () => s_doubleNextLine(indent), - () => zeroOrMore(() => consume(isSpace)) - ]); - }); - return true; - } - - // 116 - void nb_doubleMultiLine(int indent) { - nb_ns_doubleInLine(); - or([ - () => s_doubleNextLine(indent), - () => zeroOrMore(() => consume(isSpace)) - ]); - } - - // 117 - bool c_quotedQuote() => captureAs("'", () => rawString("''")); - - // 118 - bool nb_singleChar() => or([ - c_quotedQuote, - () => consume((c) => isJson(c) && c != SINGLE_QUOTE) - ]); - - // 119 - bool ns_singleChar() => !isSpace(peek()) && truth(nb_singleChar()); - - // 120 - Node c_singleQuoted(int indent, int ctx) => context('string', () { - return transaction(() { - var start = _scanner.state; - if (!truth(c_indicator(C_SINGLE_QUOTE))) return null; - var contents = nb_singleText(indent, ctx); - if (!truth(c_indicator(C_SINGLE_QUOTE))) return null; - return new ScalarNode("!", _scanner.spanFrom(start), content: contents); - }); - }); - - // 121 - String nb_singleText(int indent, int ctx) => captureString(() { - switch (ctx) { - case FLOW_OUT: - case FLOW_IN: - nb_singleMultiLine(indent); - break; - case BLOCK_KEY: - case FLOW_KEY: - nb_singleOneLine(indent); - break; - } - return true; - }); - - // 122 - void nb_singleOneLine(int indent) { - zeroOrMore(nb_singleChar); - } - - // 123 - void nb_ns_singleInLine() { - zeroOrMore(() => transaction(() { - zeroOrMore(() => consume(isSpace)); - return ns_singleChar(); - })); - } - - // 124 - bool s_singleNextLine(int indent) { - if (!truth(s_flowFolded(indent))) return false; - zeroOrOne(() { - if (!truth(ns_singleChar())) return; - nb_ns_singleInLine(); - or([ - () => s_singleNextLine(indent), - () => zeroOrMore(() => consume(isSpace)) - ]); - }); - return true; - } - - // 125 - void nb_singleMultiLine(int indent) { - nb_ns_singleInLine(); - or([ - () => s_singleNextLine(indent), - () => zeroOrMore(() => consume(isSpace)) - ]); - } - - // 126 - bool ns_plainFirst(int ctx) { - var char = peek(); - var indicator = indicatorType(char); - if (indicator == C_RESERVED) { - error("Reserved indicators can't start a plain scalar"); - } - var match = (isNonSpace(char) && indicator == null) || - ((indicator == C_MAPPING_KEY || - indicator == C_MAPPING_VALUE || - indicator == C_SEQUENCE_ENTRY) && - isPlainSafe(ctx, peek(1))); - - if (match) next(); - return match; - } - - // 127 - bool isPlainSafe(int ctx, int char) { - switch (ctx) { - case FLOW_OUT: - case BLOCK_KEY: - // 128 - return isNonSpace(char); - case FLOW_IN: - case FLOW_KEY: - // 129 - return isNonSpace(char) && !isFlowIndicator(char); - default: throw 'Invalid context "$ctx".'; - } - } - - // 130 - bool ns_plainChar(int ctx) { - var char = peek(); - var indicator = indicatorType(char); - var safeChar = isPlainSafe(ctx, char) && indicator != C_MAPPING_VALUE && - indicator != C_COMMENT; - var nonCommentHash = isNonSpace(peek(-1)) && indicator == C_COMMENT; - var nonMappingColon = indicator == C_MAPPING_VALUE && - isPlainSafe(ctx, peek(1)); - var match = safeChar || nonCommentHash || nonMappingColon; - - if (match) next(); - return match; - } - - // 131 - String ns_plain(int indent, int ctx) => context('plain scalar', () { - return captureString(() { - switch (ctx) { - case FLOW_OUT: - case FLOW_IN: - return ns_plainMultiLine(indent, ctx); - case BLOCK_KEY: - case FLOW_KEY: - return ns_plainOneLine(ctx); - default: throw 'Invalid context "$ctx".'; - } - }); - }); - - // 132 - void nb_ns_plainInLine(int ctx) { - zeroOrMore(() => transaction(() { - zeroOrMore(() => consume(isSpace)); - return ns_plainChar(ctx); - })); - } - - // 133 - bool ns_plainOneLine(int ctx) { - if (truth(c_forbidden())) return false; - if (!truth(ns_plainFirst(ctx))) return false; - nb_ns_plainInLine(ctx); - return true; - } - - // 134 - bool s_ns_plainNextLine(int indent, int ctx) => transaction(() { - if (!truth(s_flowFolded(indent))) return false; - if (truth(c_forbidden())) return false; - if (!truth(ns_plainChar(ctx))) return false; - nb_ns_plainInLine(ctx); - return true; - }); - - // 135 - bool ns_plainMultiLine(int indent, int ctx) { - if (!truth(ns_plainOneLine(ctx))) return false; - zeroOrMore(() => s_ns_plainNextLine(indent, ctx)); - return true; - } - - // 136 - int inFlow(int ctx) { - switch (ctx) { - case FLOW_OUT: - case FLOW_IN: - return FLOW_IN; - case BLOCK_KEY: - case FLOW_KEY: - return FLOW_KEY; - } - throw "unreachable"; - } - - // 137 - SequenceNode c_flowSequence(int indent, int ctx) => transaction(() { - var start = _scanner.state; - if (!truth(c_indicator(C_SEQUENCE_START))) return null; - zeroOrOne(() => s_separate(indent, ctx)); - var content = zeroOrOne(() => ns_s_flowSeqEntries(indent, inFlow(ctx))); - if (!truth(c_indicator(C_SEQUENCE_END))) return null; - return new SequenceNode("?", new List<Node>.from(content), - _scanner.spanFrom(start)); - }); - - // 138 - Iterable<Node> ns_s_flowSeqEntries(int indent, int ctx) { - var first = ns_flowSeqEntry(indent, ctx); - if (!truth(first)) return new Queue<Node>(); - zeroOrOne(() => s_separate(indent, ctx)); - - var rest; - if (truth(c_indicator(C_COLLECT_ENTRY))) { - zeroOrOne(() => s_separate(indent, ctx)); - rest = zeroOrOne(() => ns_s_flowSeqEntries(indent, ctx)); + parseTag() { + tagToken = token; + span = span.expand(token.span); + token = _scanner.advance(); } - if (rest == null) rest = new Queue<Node>(); - rest.addFirst(first); - - return rest; - } - - // 139 - Node ns_flowSeqEntry(int indent, int ctx) => or([ - () => ns_flowPair(indent, ctx), - () => ns_flowNode(indent, ctx) - ]); - - // 140 - Node c_flowMapping(int indent, int ctx) { - var start = _scanner.state; - if (!truth(c_indicator(C_MAPPING_START))) return null; - zeroOrOne(() => s_separate(indent, ctx)); - var content = zeroOrOne(() => ns_s_flowMapEntries(indent, inFlow(ctx))); - if (!truth(c_indicator(C_MAPPING_END))) return null; - return new MappingNode("?", content, _scanner.spanFrom(start)); - } - - // 141 - Map ns_s_flowMapEntries(int indent, int ctx) { - var first = ns_flowMapEntry(indent, ctx); - if (!truth(first)) return deepEqualsMap(); - zeroOrOne(() => s_separate(indent, ctx)); - - var rest; - if (truth(c_indicator(C_COLLECT_ENTRY))) { - zeroOrOne(() => s_separate(indent, ctx)); - rest = ns_s_flowMapEntries(indent, ctx); + if (token is AnchorToken) { + parseAnchor(); + if (token is TagToken) parseTag(); + } else if (token is TagToken) { + parseTag(); + if (token is AnchorToken) parseAnchor(); } - if (rest == null) rest = deepEqualsMap(); - - // TODO(nweiz): Duplicate keys should be an error. This includes keys with - // different representations but the same value (e.g. 10 vs 0xa). To make - // this user-friendly we'll probably also want to associate nodes with a - // source range. - if (!rest.containsKey(first.first)) rest[first.first] = first.last; - - return rest; - } - - // 142 - Pair<Node, Node> ns_flowMapEntry(int indent, int ctx) => or([ - () => transaction(() { - if (!truth(c_indicator(C_MAPPING_KEY))) return false; - if (!truth(s_separate(indent, ctx))) return false; - return ns_flowMapExplicitEntry(indent, ctx); - }), - () => ns_flowMapImplicitEntry(indent, ctx) - ]); - - // 143 - Pair<Node, Node> ns_flowMapExplicitEntry(int indent, int ctx) => or([ - () => ns_flowMapImplicitEntry(indent, ctx), - () => new Pair<Node, Node>(e_node(), e_node()) - ]); - - // 144 - Pair<Node, Node> ns_flowMapImplicitEntry(int indent, int ctx) => or([ - () => ns_flowMapYamlKeyEntry(indent, ctx), - () => c_ns_flowMapEmptyKeyEntry(indent, ctx), - () => c_ns_flowMapJsonKeyEntry(indent, ctx) - ]); - - // 145 - Pair<Node, Node> ns_flowMapYamlKeyEntry(int indent, int ctx) { - var key = ns_flowYamlNode(indent, ctx); - if (!truth(key)) return null; - var value = or([ - () => transaction(() { - zeroOrOne(() => s_separate(indent, ctx)); - return c_ns_flowMapSeparateValue(indent, ctx); - }), - e_node - ]); - return new Pair<Node, Node>(key, value); - } - - // 146 - Pair<Node, Node> c_ns_flowMapEmptyKeyEntry(int indent, int ctx) { - var value = c_ns_flowMapSeparateValue(indent, ctx); - if (!truth(value)) return null; - return new Pair<Node, Node>(e_node(), value); - } - - // 147 - Node c_ns_flowMapSeparateValue(int indent, int ctx) => transaction(() { - if (!truth(c_indicator(C_MAPPING_VALUE))) return null; - if (isPlainSafe(ctx, peek())) return null; - - return or([ - () => transaction(() { - if (!s_separate(indent, ctx)) return null; - return ns_flowNode(indent, ctx); - }), - e_node - ]); - }); - - // 148 - Pair<Node, Node> c_ns_flowMapJsonKeyEntry(int indent, int ctx) { - var key = c_flowJsonNode(indent, ctx); - if (!truth(key)) return null; - var value = or([ - () => transaction(() { - zeroOrOne(() => s_separate(indent, ctx)); - return c_ns_flowMapAdjacentValue(indent, ctx); - }), - e_node - ]); - return new Pair<Node, Node>(key, value); - } - - // 149 - Node c_ns_flowMapAdjacentValue(int indent, int ctx) { - if (!truth(c_indicator(C_MAPPING_VALUE))) return null; - return or([ - () => transaction(() { - zeroOrOne(() => s_separate(indent, ctx)); - return ns_flowNode(indent, ctx); - }), - e_node - ]); - } - - // 150 - Node ns_flowPair(int indent, int ctx) { - var start = _scanner.state; - var pair = or([ - () => transaction(() { - if (!truth(c_indicator(C_MAPPING_KEY))) return null; - if (!truth(s_separate(indent, ctx))) return null; - return ns_flowMapExplicitEntry(indent, ctx); - }), - () => ns_flowPairEntry(indent, ctx) - ]); - if (!truth(pair)) return null; - - return map([pair], _scanner.spanFrom(start)); - } - - // 151 - Pair<Node, Node> ns_flowPairEntry(int indent, int ctx) => or([ - () => ns_flowPairYamlKeyEntry(indent, ctx), - () => c_ns_flowMapEmptyKeyEntry(indent, ctx), - () => c_ns_flowPairJsonKeyEntry(indent, ctx) - ]); - - // 152 - Pair<Node, Node> ns_flowPairYamlKeyEntry(int indent, int ctx) => - transaction(() { - var key = ns_s_implicitYamlKey(FLOW_KEY); - if (!truth(key)) return null; - var value = c_ns_flowMapSeparateValue(indent, ctx); - if (!truth(value)) return null; - return new Pair<Node, Node>(key, value); - }); - - // 153 - Pair<Node, Node> c_ns_flowPairJsonKeyEntry(int indent, int ctx) => - transaction(() { - var key = c_s_implicitJsonKey(FLOW_KEY); - if (!truth(key)) return null; - var value = c_ns_flowMapAdjacentValue(indent, ctx); - if (!truth(value)) return null; - return new Pair<Node, Node>(key, value); - }); - - // 154 - Node ns_s_implicitYamlKey(int ctx) => transaction(() { - // TODO(nweiz): this is supposed to be limited to 1024 characters. - - // The indentation parameter is "null" since it's unused in this path - var node = ns_flowYamlNode(null, ctx); - if (!truth(node)) return null; - zeroOrOne(s_separateInLine); - return node; - }); - - // 155 - Node c_s_implicitJsonKey(int ctx) => transaction(() { - // TODO(nweiz): this is supposed to be limited to 1024 characters. - - // The indentation parameter is "null" since it's unused in this path - var node = c_flowJsonNode(null, ctx); - if (!truth(node)) return null; - zeroOrOne(s_separateInLine); - return node; - }); - - // 156 - Node ns_flowYamlContent(int indent, int ctx) { - var start = _scanner.state; - var str = ns_plain(indent, ctx); - if (!truth(str)) return null; - return new ScalarNode("?", _scanner.spanFrom(start), content: str); - } - - // 157 - Node c_flowJsonContent(int indent, int ctx) => or([ - () => c_flowSequence(indent, ctx), - () => c_flowMapping(indent, ctx), - () => c_singleQuoted(indent, ctx), - () => c_doubleQuoted(indent, ctx) - ]); - - // 158 - Node ns_flowContent(int indent, int ctx) => or([ - () => ns_flowYamlContent(indent, ctx), - () => c_flowJsonContent(indent, ctx) - ]); - - // 159 - Node ns_flowYamlNode(int indent, int ctx) => or([ - c_ns_aliasNode, - () => ns_flowYamlContent(indent, ctx), - () { - var props = c_ns_properties(indent, ctx); - if (!truth(props)) return null; - var node = or([ - () => transaction(() { - if (!truth(s_separate(indent, ctx))) return null; - return ns_flowYamlContent(indent, ctx); - }), - e_scalar - ]); - return addProps(node, props); - } - ]); - - // 160 - Node c_flowJsonNode(int indent, int ctx) => transaction(() { - var props; - zeroOrOne(() => transaction(() { - props = c_ns_properties(indent, ctx); - if (!truth(props)) return null; - return s_separate(indent, ctx); - })); - - return addProps(c_flowJsonContent(indent, ctx), props); - }); - - // 161 - Node ns_flowNode(int indent, int ctx) => or([ - c_ns_aliasNode, - () => ns_flowContent(indent, ctx), - () => transaction(() { - var props = c_ns_properties(indent, ctx); - if (!truth(props)) return null; - var node = or([ - () => transaction(() => s_separate(indent, ctx) ? - ns_flowContent(indent, ctx) : null), - e_scalar]); - return addProps(node, props); - }) - ]); - - // 162 - _BlockHeader c_b_blockHeader() => transaction(() { - var indentation = c_indentationIndicator(); - var chomping = c_chompingIndicator(); - if (!truth(indentation)) indentation = c_indentationIndicator(); - if (!truth(s_b_comment())) return null; - - return new _BlockHeader(indentation, chomping); - }); - - // 163 - int c_indentationIndicator() { - if (!isDecDigit(peek())) return null; - return next() - NUMBER_0; - } - - // 164 - int c_chompingIndicator() { - switch (peek()) { - case HYPHEN: - next(); - return CHOMPING_STRIP; - case PLUS: - next(); - return CHOMPING_KEEP; - default: - return CHOMPING_CLIP; - } - } - - // 165 - bool b_chompedLast(int chomping) { - if (_scanner.isDone) return true; - switch (chomping) { - case CHOMPING_STRIP: - return b_nonContent(); - case CHOMPING_CLIP: - case CHOMPING_KEEP: - return b_asLineFeed(); - } - throw "unreachable"; - } - - // 166 - void l_chompedEmpty(int indent, int chomping) { - switch (chomping) { - case CHOMPING_STRIP: - case CHOMPING_CLIP: - l_stripEmpty(indent); - break; - case CHOMPING_KEEP: - l_keepEmpty(indent); - break; - } - } - - // 167 - void l_stripEmpty(int indent) { - captureAs('', () { - zeroOrMore(() => transaction(() { - if (!truth(s_indentLessThanOrEqualTo(indent))) return false; - return b_nonContent(); - })); - zeroOrOne(() => l_trailComments(indent)); - return true; - }); - } - - // 168 - void l_keepEmpty(int indent) { - zeroOrMore(() => captureAs('\n', () => l_empty(indent, BLOCK_IN))); - zeroOrOne(() => captureAs('', () => l_trailComments(indent))); - } - - // 169 - bool l_trailComments(int indent) => transaction(() { - if (!truth(s_indentLessThanOrEqualTo(indent))) return false; - if (!truth(c_nb_commentText())) return false; - if (!truth(b_comment())) return false; - zeroOrMore(l_comment); - return true; - }); - - // 170 - Node c_l_literal(int indent) => transaction(() { - var start = _scanner.state; - if (!truth(c_indicator(C_LITERAL))) return null; - var header = c_b_blockHeader(); - if (!truth(header)) return null; - - var additionalIndent = blockScalarAdditionalIndentation(header, indent); - var content = l_literalContent(indent + additionalIndent, header.chomping); - if (!truth(content)) return null; - - return new ScalarNode("!", _scanner.spanFrom(start), content: content); - }); - - // 171 - bool l_nb_literalText(int indent) => transaction(() { - zeroOrMore(() => captureAs("\n", () => l_empty(indent, BLOCK_IN))); - if (!truth(captureAs("", () => s_indent(indent)))) return false; - return truth(oneOrMore(() => consume(isNonBreak))); - }); - - // 172 - bool b_nb_literalNext(int indent) => transaction(() { - if (!truth(b_asLineFeed())) return false; - return l_nb_literalText(indent); - }); - - // 173 - String l_literalContent(int indent, int chomping) => captureString(() { - transaction(() { - if (!truth(l_nb_literalText(indent))) return false; - zeroOrMore(() => b_nb_literalNext(indent)); - return b_chompedLast(chomping); - }); - l_chompedEmpty(indent, chomping); - return true; - }); - - // 174 - Node c_l_folded(int indent) => transaction(() { - var start = _scanner.state; - if (!truth(c_indicator(C_FOLDED))) return null; - var header = c_b_blockHeader(); - if (!truth(header)) return null; - - var additionalIndent = blockScalarAdditionalIndentation(header, indent); - var content = l_foldedContent(indent + additionalIndent, header.chomping); - if (!truth(content)) return null; - - return new ScalarNode("!", _scanner.spanFrom(start), content: content); - }); - - // 175 - bool s_nb_foldedText(int indent) => transaction(() { - if (!truth(captureAs('', () => s_indent(indent)))) return false; - if (!truth(consume(isNonSpace))) return false; - zeroOrMore(() => consume(isNonBreak)); - return true; - }); - - // 176 - bool l_nb_foldedLines(int indent) { - if (!truth(s_nb_foldedText(indent))) return false; - zeroOrMore(() => transaction(() { - if (!truth(b_l_folded(indent, BLOCK_IN))) return false; - return s_nb_foldedText(indent); - })); - return true; - } - - // 177 - bool s_nb_spacedText(int indent) => transaction(() { - if (!truth(captureAs('', () => s_indent(indent)))) return false; - if (!truth(consume(isSpace))) return false; - zeroOrMore(() => consume(isNonBreak)); - return true; - }); - - // 178 - bool b_l_spaced(int indent) { - if (!truth(b_asLineFeed())) return false; - zeroOrMore(() => captureAs("\n", () => l_empty(indent, BLOCK_IN))); - return true; - } - - // 179 - bool l_nb_spacedLines(int indent) { - if (!truth(s_nb_spacedText(indent))) return false; - zeroOrMore(() => transaction(() { - if (!truth(b_l_spaced(indent))) return false; - return s_nb_spacedText(indent); - })); - return true; - } - - // 180 - bool l_nb_sameLines(int indent) => transaction(() { - zeroOrMore(() => captureAs('\n', () => l_empty(indent, BLOCK_IN))); - return or([ - () => l_nb_foldedLines(indent), - () => l_nb_spacedLines(indent) - ]); - }); - - // 181 - bool l_nb_diffLines(int indent) { - if (!truth(l_nb_sameLines(indent))) return false; - zeroOrMore(() => transaction(() { - if (!truth(b_asLineFeed())) return false; - return l_nb_sameLines(indent); - })); - return true; - } - - // 182 - String l_foldedContent(int indent, int chomping) => captureString(() { - transaction(() { - if (!truth(l_nb_diffLines(indent))) return false; - return b_chompedLast(chomping); - }); - l_chompedEmpty(indent, chomping); - return true; - }); - - // 183 - SequenceNode l_blockSequence(int indent) => context('sequence', () { - var additionalIndent = countIndentation() - indent; - if (additionalIndent <= 0) return null; - - var start = _scanner.state; - var content = oneOrMore(() => transaction(() { - if (!truth(s_indent(indent + additionalIndent))) return null; - return c_l_blockSeqEntry(indent + additionalIndent); - })); - if (!truth(content)) return null; - - return new SequenceNode("?", content, _scanner.spanFrom(start)); - }); - - // 184 - Node c_l_blockSeqEntry(int indent) => transaction(() { - if (!truth(c_indicator(C_SEQUENCE_ENTRY))) return null; - if (isNonSpace(peek())) return null; - - return s_l_blockIndented(indent, BLOCK_IN); - }); - - // 185 - Node s_l_blockIndented(int indent, int ctx) { - var additionalIndent = countIndentation(); - return or([ - () => transaction(() { - if (!truth(s_indent(additionalIndent))) return null; - return or([ - () => ns_l_compactSequence(indent + 1 + additionalIndent), - () => ns_l_compactMapping(indent + 1 + additionalIndent)]); - }), - () => s_l_blockNode(indent, ctx), - () => s_l_comments() ? e_node() : null]); - } - - // 186 - Node ns_l_compactSequence(int indent) => context('sequence', () { - var start = _scanner.state; - var first = c_l_blockSeqEntry(indent); - if (!truth(first)) return null; - - var content = zeroOrMore(() => transaction(() { - if (!truth(s_indent(indent))) return null; - return c_l_blockSeqEntry(indent); - })); - content.insert(0, first); - - return new SequenceNode("?", content, _scanner.spanFrom(start)); - }); - - // 187 - Node l_blockMapping(int indent) => context('mapping', () { - var additionalIndent = countIndentation() - indent; - if (additionalIndent <= 0) return null; - - var start = _scanner.state; - var pairs = oneOrMore(() => transaction(() { - if (!truth(s_indent(indent + additionalIndent))) return null; - return ns_l_blockMapEntry(indent + additionalIndent); - })); - if (!truth(pairs)) return null; - - return map(pairs, _scanner.spanFrom(start)); - }); - - // 188 - Pair<Node, Node> ns_l_blockMapEntry(int indent) => or([ - () => c_l_blockMapExplicitEntry(indent), - () => ns_l_blockMapImplicitEntry(indent) - ]); - - // 189 - Pair<Node, Node> c_l_blockMapExplicitEntry(int indent) { - var key = c_l_blockMapExplicitKey(indent); - if (!truth(key)) return null; - - var value = or([ - () => l_blockMapExplicitValue(indent), - e_node - ]); - - return new Pair<Node, Node>(key, value); - } - - // 190 - Node c_l_blockMapExplicitKey(int indent) => transaction(() { - if (!truth(c_indicator(C_MAPPING_KEY))) return null; - return s_l_blockIndented(indent, BLOCK_OUT); - }); - - // 191 - Node l_blockMapExplicitValue(int indent) => transaction(() { - if (!truth(s_indent(indent))) return null; - if (!truth(c_indicator(C_MAPPING_VALUE))) return null; - return s_l_blockIndented(indent, BLOCK_OUT); - }); - - // 192 - Pair<Node, Node> ns_l_blockMapImplicitEntry(int indent) => transaction(() { - var key = or([ns_s_blockMapImplicitKey, e_node]); - var value = c_l_blockMapImplicitValue(indent); - return truth(value) ? new Pair<Node, Node>(key, value) : null; - }); - - // 193 - Node ns_s_blockMapImplicitKey() => context('mapping key', () => or([ - () => c_s_implicitJsonKey(BLOCK_KEY), - () => ns_s_implicitYamlKey(BLOCK_KEY) - ])); - - // 194 - Node c_l_blockMapImplicitValue(int indent) => context('mapping value', () => - transaction(() { - if (!truth(c_indicator(C_MAPPING_VALUE))) return null; - return or([ - () => s_l_blockNode(indent, BLOCK_OUT), - () => s_l_comments() ? e_node() : null - ]); - })); - - // 195 - Node ns_l_compactMapping(int indent) => context('mapping', () { - var start = _scanner.state; - var first = ns_l_blockMapEntry(indent); - if (!truth(first)) return null; - - var pairs = zeroOrMore(() => transaction(() { - if (!truth(s_indent(indent))) return null; - return ns_l_blockMapEntry(indent); - })); - pairs.insert(0, first); - - return map(pairs, _scanner.spanFrom(start)); - }); - - // 196 - Node s_l_blockNode(int indent, int ctx) => or([ - () => s_l_blockInBlock(indent, ctx), - () => s_l_flowInBlock(indent) - ]); - - // 197 - Node s_l_flowInBlock(int indent) => transaction(() { - if (!truth(s_separate(indent + 1, FLOW_OUT))) return null; - var node = ns_flowNode(indent + 1, FLOW_OUT); - if (!truth(node)) return null; - if (!truth(s_l_comments())) return null; - return node; - }); - - // 198 - Node s_l_blockInBlock(int indent, int ctx) => or([ - () => s_l_blockScalar(indent, ctx), - () => s_l_blockCollection(indent, ctx) - ]); - - // 199 - Node s_l_blockScalar(int indent, int ctx) => transaction(() { - if (!truth(s_separate(indent + 1, ctx))) return null; - var props = transaction(() { - var innerProps = c_ns_properties(indent + 1, ctx); - if (!truth(innerProps)) return null; - if (!truth(s_separate(indent + 1, ctx))) return null; - return innerProps; - }); - - var node = or([() => c_l_literal(indent), () => c_l_folded(indent)]); - if (!truth(node)) return null; - return addProps(node, props); - }); - - // 200 - Node s_l_blockCollection(int indent, int ctx) => transaction(() { - var props = transaction(() { - if (!truth(s_separate(indent + 1, ctx))) return null; - return c_ns_properties(indent + 1, ctx); - }); - - if (!truth(s_l_comments())) return null; - return or([ - () => l_blockSequence(seqSpaces(indent, ctx)), - () => l_blockMapping(indent)]); - }); - - // 201 - int seqSpaces(int indent, int ctx) => ctx == BLOCK_OUT ? indent - 1 : indent; - - // 202 - void l_documentPrefix() { - zeroOrMore(l_comment); - } - - // 203 - bool c_directivesEnd() => rawString("---"); - - // 204 - bool c_documentEnd() => rawString("..."); - - // 205 - bool l_documentSuffix() => transaction(() { - if (!truth(c_documentEnd())) return false; - return s_l_comments(); - }); - - // 206 - bool c_forbidden() { - if (!_inBareDocument || !atStartOfLine) return false; - var forbidden = false; - transaction(() { - if (!truth(or([c_directivesEnd, c_documentEnd]))) return; - var char = peek(); - forbidden = isBreak(char) || isSpace(char) || _scanner.isDone; - return; - }); - return forbidden; - } - - // 207 - Node l_bareDocument() { - try { - _inBareDocument = true; - return s_l_blockNode(-1, BLOCK_IN); - } finally { - _inBareDocument = false; - } - } - - // 208 - Node l_explicitDocument() { - if (!truth(c_directivesEnd())) return null; - var doc = l_bareDocument(); - if (truth(doc)) return doc; - - doc = e_node(); - s_l_comments(); - return doc; - } - - // 209 - Node l_directiveDocument() { - if (!truth(oneOrMore(l_directive))) return null; - var doc = l_explicitDocument(); - if (doc != null) return doc; - parseFailed(); - return null; // Unreachable. - } - - // 210 - Node l_anyDocument() => - or([l_directiveDocument, l_explicitDocument, l_bareDocument]); - - // 211 - Pair<List<Node>, SourceSpan> l_yamlStream() { - var start = _scanner.state; - var docs = []; - zeroOrMore(l_documentPrefix); - var first = zeroOrOne(l_anyDocument); - if (!truth(first)) first = e_node(); - docs.add(first); - - zeroOrMore(() { - var doc; - if (truth(oneOrMore(l_documentSuffix))) { - zeroOrMore(l_documentPrefix); - doc = zeroOrOne(l_anyDocument); + var tag; + if (tagToken != null) { + if (tagToken.handle == null) { + tag = tagToken.suffix; } else { - zeroOrMore(l_documentPrefix); - doc = zeroOrOne(l_explicitDocument); + var tagDirective = _tagDirectives[tagToken.handle]; + if (tagDirective == null) { + throw new YamlException("Undefined tag handle.", tagToken.span); + } + + tag = tagDirective.prefix + tagToken.suffix; } - if (truth(doc)) docs.add(doc); - return doc; - }); - - if (!_scanner.isDone) parseFailed(); - return new Pair(docs, _scanner.spanFrom(start)); - } -} - -/// The information in the header for a block scalar. -class _BlockHeader { - final int additionalIndent; - final int chomping; - - _BlockHeader(this.additionalIndent, this.chomping); - - bool get autoDetectIndent => additionalIndent == null; -} - -/// A range of characters in the YAML document, from [start] to [end] -/// (inclusive). -class _Range { - /// The first character in the range. - final int start; - - /// The last character in the range. - final int end; - - _Range(this.start, this.end); - - /// Returns whether or not [pos] lies within this range. - bool contains(int pos) => pos >= start && pos <= end; -} - -/// A map that associates [E] values with [_Range]s. It's efficient to create -/// new associations, but finding the value associated with a position is more -/// expensive. -class _RangeMap<E> { - /// The ranges and their associated elements. - final List<Pair<_Range, E>> _contents = <Pair<_Range, E>>[]; - - _RangeMap(); - - /// Returns the value associated with the range in which [pos] lies, or null - /// if there is no such range. If there's more than one such range, the most - /// recently set one is used. - E operator[](int pos) { - // Iterate backwards through contents so the more recent range takes - // precedence. - for (var pair in _contents.reversed) { - if (pair.first.contains(pos)) return pair.last; } - return null; + + if (indentlessSequence && token.type == TokenType.BLOCK_ENTRY) { + _state = _State.INDENTLESS_SEQUENCE_ENTRY; + return new SequenceStartEvent( + span.expand(token.span), CollectionStyle.BLOCK, + anchor: anchor, tag: tag); + } + + if (token is ScalarToken) { + // All non-plain scalars have the "!" tag by default. + if (tag == null && token.style != ScalarStyle.PLAIN) tag = "!"; + + _state = _states.removeLast(); + _scanner.scan(); + return new ScalarEvent( + span.expand(token.span), token.value, token.style, + anchor: anchor, tag: tag); + } + + if (token.type == TokenType.FLOW_SEQUENCE_START) { + _state = _State.FLOW_SEQUENCE_FIRST_ENTRY; + return new SequenceStartEvent( + span.expand(token.span), CollectionStyle.FLOW, + anchor: anchor, tag: tag); + } + + if (token.type == TokenType.FLOW_MAPPING_START) { + _state = _State.FLOW_MAPPING_FIRST_KEY; + return new MappingStartEvent( + span.expand(token.span), CollectionStyle.FLOW, + anchor: anchor, tag: tag); + } + + if (block && token.type == TokenType.BLOCK_SEQUENCE_START) { + _state = _State.BLOCK_SEQUENCE_FIRST_ENTRY; + return new SequenceStartEvent( + span.expand(token.span), CollectionStyle.BLOCK, + anchor: anchor, tag: tag); + } + + + if (block && token.type == TokenType.BLOCK_MAPPING_START) { + _state = _State.BLOCK_MAPPING_FIRST_KEY; + return new MappingStartEvent( + span.expand(token.span), CollectionStyle.BLOCK, + anchor: anchor, tag: tag); + } + + if (anchor != null || tag != null) { + _state = _states.removeLast(); + return new ScalarEvent( + span, '', ScalarStyle.PLAIN, + anchor: anchor, tag: tag); + } + + throw new YamlException("Expected node content.", span); } - /// Associates [value] with [range]. - operator[]=(_Range range, E value) => - _contents.add(new Pair<_Range, E>(range, value)); + /// Parses the productions: + /// + /// block_sequence ::= + /// BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END + /// ******************** *********** * ********* + Event _parseBlockSequenceEntry() { + var token = _scanner.peek(); + + if (token.type == TokenType.BLOCK_ENTRY) { + token = _scanner.advance(); + + if (token.type == TokenType.BLOCK_ENTRY || + token.type == TokenType.BLOCK_END) { + _state = _State.BLOCK_SEQUENCE_ENTRY; + return _processEmptyScalar(token.span.end); + } else { + _states.add(_State.BLOCK_SEQUENCE_ENTRY); + return _parseNode(block: true); + } + } + + if (token.type == TokenType.BLOCK_END) { + _scanner.scan(); + _state = _states.removeLast(); + return new Event(EventType.SEQUENCE_END, token.span); + } + + throw new YamlException("While parsing a block collection, expected '-'.", + token.span.start.pointSpan()); + } + + /// Parses the productions: + /// + /// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ + /// *********** * + Event _parseIndentlessSequenceEntry() { + var token = _scanner.peek(); + + if (token.type != TokenType.BLOCK_ENTRY) { + _state = _states.removeLast(); + return new Event(EventType.SEQUENCE_END, token.span.start.pointSpan()); + } + + var start = token.span.start; + token = _scanner.advance(); + + if (token.type == TokenType.BLOCK_ENTRY || + token.type == TokenType.KEY || + token.type == TokenType.VALUE || + token.type == TokenType.BLOCK_END) { + _state = _State.INDENTLESS_SEQUENCE_ENTRY; + return _processEmptyScalar(start); + } else { + _states.add(_State.INDENTLESS_SEQUENCE_ENTRY); + return _parseNode(block: true); + } + } + + /// Parses the productions: + /// + /// block_mapping ::= BLOCK-MAPPING_START + /// ******************* + /// ((KEY block_node_or_indentless_sequence?)? + /// *** * + /// (VALUE block_node_or_indentless_sequence?)?)* + /// + /// BLOCK-END + /// ********* + Event _parseBlockMappingKey() { + var token = _scanner.peek(); + if (token.type == TokenType.KEY) { + var start = token.span.start; + token = _scanner.advance(); + + if (token.type == TokenType.KEY || + token.type == TokenType.VALUE || + token.type == TokenType.BLOCK_END) { + _state = _State.BLOCK_MAPPING_VALUE; + return _processEmptyScalar(start); + } else { + _states.add(_State.BLOCK_MAPPING_VALUE); + return _parseNode(block: true, indentlessSequence: true); + } + } + + // libyaml doesn't allow empty keys without an explicit key indicator, but + // the spec does. See example 8.18: + // http://yaml.org/spec/1.2/spec.html#id2798896. + if (token.type == TokenType.VALUE) { + _state = _State.BLOCK_MAPPING_VALUE; + return _processEmptyScalar(token.span.start); + } + + if (token.type == TokenType.BLOCK_END) { + _scanner.scan(); + _state = _states.removeLast(); + return new Event(EventType.MAPPING_END, token.span); + } + + throw new YamlException("Expected a key while parsing a block mapping.", + token.span.start.pointSpan()); + } + + /// Parses the productions: + /// + /// block_mapping ::= BLOCK-MAPPING_START + /// + /// ((KEY block_node_or_indentless_sequence?)? + /// + /// (VALUE block_node_or_indentless_sequence?)?)* + /// ***** * + /// BLOCK-END + /// + Event _parseBlockMappingValue() { + var token = _scanner.peek(); + + if (token.type != TokenType.VALUE) { + _state = _State.BLOCK_MAPPING_KEY; + return _processEmptyScalar(token.span.start); + } + + var start = token.span.start; + token = _scanner.advance(); + if (token.type == TokenType.KEY || + token.type == TokenType.VALUE || + token.type == TokenType.BLOCK_END) { + _state = _State.BLOCK_MAPPING_KEY; + return _processEmptyScalar(start); + } else { + _states.add(_State.BLOCK_MAPPING_KEY); + return _parseNode(block: true, indentlessSequence: true); + } + } + + /// Parses the productions: + /// + /// flow_sequence ::= FLOW-SEQUENCE-START + /// ******************* + /// (flow_sequence_entry FLOW-ENTRY)* + /// * ********** + /// flow_sequence_entry? + /// * + /// FLOW-SEQUENCE-END + /// ***************** + /// flow_sequence_entry ::= + /// flow_node | KEY flow_node? (VALUE flow_node?)? + /// * + Event _parseFlowSequenceEntry({bool first: false}) { + if (first) _scanner.scan(); + var token = _scanner.peek(); + + if (token.type != TokenType.FLOW_SEQUENCE_END) { + if (!first) { + if (token.type != TokenType.FLOW_ENTRY) { + throw new YamlException( + "While parsing a flow sequence, expected ',' or ']'.", + token.span.start.pointSpan()); + } + + token = _scanner.advance(); + } + + if (token.type == TokenType.KEY) { + _state = _State.FLOW_SEQUENCE_ENTRY_MAPPING_KEY; + _scanner.scan(); + return new MappingStartEvent( + token.span, CollectionStyle.FLOW); + } else if (token.type != TokenType.FLOW_SEQUENCE_END) { + _states.add(_State.FLOW_SEQUENCE_ENTRY); + return _parseNode(); + } + } + + _scanner.scan(); + _state = _states.removeLast(); + return new Event(EventType.SEQUENCE_END, token.span); + } + + /// Parses the productions: + /// + /// flow_sequence_entry ::= + /// flow_node | KEY flow_node? (VALUE flow_node?)? + /// *** * + Event _parseFlowSequenceEntryMappingKey() { + var token = _scanner.peek(); + + if (token.type == TokenType.VALUE || + token.type == TokenType.FLOW_ENTRY || + token.type == TokenType.FLOW_SEQUENCE_END) { + // libyaml consumes the token here, but that seems like a bug, since it + // always causes [_parseFlowSequenceEntryMappingValue] to emit an empty + // scalar. + + var start = token.span.start; + _state = _State.FLOW_SEQUENCE_ENTRY_MAPPING_VALUE; + return _processEmptyScalar(start); + } else { + _states.add(_State.FLOW_SEQUENCE_ENTRY_MAPPING_VALUE); + return _parseNode(); + } + } + + /// Parses the productions: + /// + /// flow_sequence_entry ::= + /// flow_node | KEY flow_node? (VALUE flow_node?)? + /// ***** * + Event _parseFlowSequenceEntryMappingValue() { + var token = _scanner.peek(); + + if (token.type == TokenType.VALUE) { + token = _scanner.advance(); + if (token.type != TokenType.FLOW_ENTRY && + token.type != TokenType.FLOW_SEQUENCE_END) { + _states.add(_State.FLOW_SEQUENCE_ENTRY_MAPPING_END); + return _parseNode(); + } + } + + _state = _State.FLOW_SEQUENCE_ENTRY_MAPPING_END; + return _processEmptyScalar(token.span.start); + } + + /// Parses the productions: + /// + /// flow_sequence_entry ::= + /// flow_node | KEY flow_node? (VALUE flow_node?)? + /// * + Event _parseFlowSequenceEntryMappingEnd() { + _state = _State.FLOW_SEQUENCE_ENTRY; + return new Event(EventType.MAPPING_END, + _scanner.peek().span.start.pointSpan()); + } + + /// Parses the productions: + /// + /// flow_mapping ::= FLOW-MAPPING-START + /// ****************** + /// (flow_mapping_entry FLOW-ENTRY)* + /// * ********** + /// flow_mapping_entry? + /// ****************** + /// FLOW-MAPPING-END + /// **************** + /// flow_mapping_entry ::= + /// flow_node | KEY flow_node? (VALUE flow_node?)? + /// * *** * + Event _parseFlowMappingKey({bool first: false}) { + if (first) _scanner.scan(); + var token = _scanner.peek(); + + if (token.type != TokenType.FLOW_MAPPING_END) { + if (!first) { + if (token.type != TokenType.FLOW_ENTRY) { + throw new YamlException( + "While parsing a flow mapping, expected ',' or '}'.", + token.span.start.pointSpan()); + } + + token = _scanner.advance(); + } + + if (token.type == TokenType.KEY) { + token = _scanner.advance(); + if (token.type != TokenType.VALUE && + token.type != TokenType.FLOW_ENTRY && + token.type != TokenType.FLOW_MAPPING_END) { + _states.add(_State.FLOW_MAPPING_VALUE); + return _parseNode(); + } else { + _state = _State.FLOW_MAPPING_VALUE; + return _processEmptyScalar(token.span.start); + } + } else if (token.type != TokenType.FLOW_MAPPING_END) { + _states.add(_State.FLOW_MAPPING_EMPTY_VALUE); + return _parseNode(); + } + } + + _scanner.scan(); + _state = _states.removeLast(); + return new Event(EventType.MAPPING_END, token.span); + } + + /// Parses the productions: + /// + /// flow_mapping_entry ::= + /// flow_node | KEY flow_node? (VALUE flow_node?)? + /// * ***** * + Event _parseFlowMappingValue({bool empty: false}) { + var token = _scanner.peek(); + + if (empty) { + _state = _State.FLOW_MAPPING_KEY; + return _processEmptyScalar(token.span.start); + } + + if (token.type == TokenType.VALUE) { + token = _scanner.advance(); + if (token.type != TokenType.FLOW_ENTRY && + token.type != TokenType.FLOW_MAPPING_END) { + _states.add(_State.FLOW_MAPPING_KEY); + return _parseNode(); + } + } + + _state = _State.FLOW_MAPPING_KEY; + return _processEmptyScalar(token.span.start); + } + + /// Generate an empty scalar event. + Event _processEmptyScalar(SourceLocation location) => + new ScalarEvent(location.pointSpan(), '', ScalarStyle.PLAIN); + + /// Parses directives. + Pair<VersionDirective, List<TagDirective>> _processDirectives() { + var token = _scanner.peek(); + + var versionDirective; + var tagDirectives = []; + var reservedDirectives = []; + while (token.type == TokenType.VERSION_DIRECTIVE || + token.type == TokenType.TAG_DIRECTIVE) { + if (token is VersionDirectiveToken) { + if (versionDirective != null) { + throw new YamlException("Duplicate %YAML directive.", token.span); + } + + if (token.major != 1 || token.minor == 0) { + throw new YamlException( + "Incompatible YAML document. This parser only supports YAML 1.1 " + "and 1.2.", + token.span); + } else if (token.minor > 2) { + // TODO(nweiz): Print to stderr when issue 6943 is fixed and dart:io + // is available. + warn("Warning: this parser only supports YAML 1.1 and 1.2.", + token.span); + } + + versionDirective = new VersionDirective(token.major, token.minor); + } else if (token is TagDirectiveToken) { + var tagDirective = new TagDirective(token.handle, token.prefix); + _appendTagDirective(tagDirective, token.span); + tagDirectives.add(tagDirective); + } + + token = _scanner.advance(); + } + + _appendTagDirective( + new TagDirective("!", "!"), + token.span.start.pointSpan(), + allowDuplicates: true); + _appendTagDirective( + new TagDirective("!!", "tag:yaml.org,2002:"), + token.span.start.pointSpan(), + allowDuplicates: true); + + return new Pair(versionDirective, tagDirectives); + } + + /// Adds a tag directive to the directives stack. + void _appendTagDirective(TagDirective newDirective, FileSpan span, + {bool allowDuplicates: false}) { + if (_tagDirectives.containsKey(newDirective.handle)) { + if (allowDuplicates) return; + throw new YamlException("Duplicate %TAG directive.", span); + } + + _tagDirectives[newDirective.handle] = newDirective; + } +} + +/// The possible states for the parser. +class _State { + /// Expect [TokenType.STREAM_START]. + static const STREAM_START = const _State("STREAM_START"); + + /// Expect the beginning of an implicit document. + static const IMPLICIT_DOCUMENT_START = + const _State("IMPLICIT_DOCUMENT_START"); + + /// Expect [TokenType.DOCUMENT_START]. + static const DOCUMENT_START = const _State("DOCUMENT_START"); + + /// Expect the content of a document. + static const DOCUMENT_CONTENT = const _State("DOCUMENT_CONTENT"); + + /// Expect [TokenType.DOCUMENT_END]. + static const DOCUMENT_END = const _State("DOCUMENT_END"); + + /// Expect a block node. + static const BLOCK_NODE = const _State("BLOCK_NODE"); + + /// Expect a block node or indentless sequence. + static const BLOCK_NODE_OR_INDENTLESS_SEQUENCE = + const _State("BLOCK_NODE_OR_INDENTLESS_SEQUENCE"); + + /// Expect a flow node. + static const FLOW_NODE = const _State("FLOW_NODE"); + + /// Expect the first entry of a block sequence. + static const BLOCK_SEQUENCE_FIRST_ENTRY = + const _State("BLOCK_SEQUENCE_FIRST_ENTRY"); + + /// Expect an entry of a block sequence. + static const BLOCK_SEQUENCE_ENTRY = const _State("BLOCK_SEQUENCE_ENTRY"); + + /// Expect an entry of an indentless sequence. + static const INDENTLESS_SEQUENCE_ENTRY = + const _State("INDENTLESS_SEQUENCE_ENTRY"); + + /// Expect the first key of a block mapping. + static const BLOCK_MAPPING_FIRST_KEY = + const _State("BLOCK_MAPPING_FIRST_KEY"); + + /// Expect a block mapping key. + static const BLOCK_MAPPING_KEY = const _State("BLOCK_MAPPING_KEY"); + + /// Expect a block mapping value. + static const BLOCK_MAPPING_VALUE = const _State("BLOCK_MAPPING_VALUE"); + + /// Expect the first entry of a flow sequence. + static const FLOW_SEQUENCE_FIRST_ENTRY = + const _State("FLOW_SEQUENCE_FIRST_ENTRY"); + + /// Expect an entry of a flow sequence. + static const FLOW_SEQUENCE_ENTRY = const _State("FLOW_SEQUENCE_ENTRY"); + + /// Expect a key of an ordered mapping. + static const FLOW_SEQUENCE_ENTRY_MAPPING_KEY = + const _State("FLOW_SEQUENCE_ENTRY_MAPPING_KEY"); + + /// Expect a value of an ordered mapping. + static const FLOW_SEQUENCE_ENTRY_MAPPING_VALUE = + const _State("FLOW_SEQUENCE_ENTRY_MAPPING_VALUE"); + + /// Expect the and of an ordered mapping entry. + static const FLOW_SEQUENCE_ENTRY_MAPPING_END = + const _State("FLOW_SEQUENCE_ENTRY_MAPPING_END"); + + /// Expect the first key of a flow mapping. + static const FLOW_MAPPING_FIRST_KEY = const _State("FLOW_MAPPING_FIRST_KEY"); + + /// Expect a key of a flow mapping. + static const FLOW_MAPPING_KEY = const _State("FLOW_MAPPING_KEY"); + + /// Expect a value of a flow mapping. + static const FLOW_MAPPING_VALUE = const _State("FLOW_MAPPING_VALUE"); + + /// Expect an empty value of a flow mapping. + static const FLOW_MAPPING_EMPTY_VALUE = + const _State("FLOW_MAPPING_EMPTY_VALUE"); + + /// Expect nothing. + static const END = const _State("END"); + + final String name; + + const _State(this.name); + + String toString() => name; }
diff --git a/pkgs/yaml/lib/src/scanner.dart b/pkgs/yaml/lib/src/scanner.dart new file mode 100644 index 0000000..0068553 --- /dev/null +++ b/pkgs/yaml/lib/src/scanner.dart
@@ -0,0 +1,1663 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library yaml.scanner; + +import 'package:collection/collection.dart'; +import 'package:string_scanner/string_scanner.dart'; +import 'package:source_span/source_span.dart'; + +import 'style.dart'; +import 'token.dart'; +import 'utils.dart'; +import 'yaml_exception.dart'; + +/// A scanner that reads a string of Unicode characters and emits [Token]s. +/// +/// This is based on the libyaml scanner, available at +/// https://github.com/yaml/libyaml/blob/master/src/scanner.c. The license for +/// that is available in ../../libyaml-license.txt. +class Scanner { + static const TAB = 0x9; + static const LF = 0xA; + static const CR = 0xD; + static const SP = 0x20; + static const DOLLAR = 0x24; + static const LEFT_PAREN = 0x28; + static const RIGHT_PAREN = 0x29; + static const PLUS = 0x2B; + static const COMMA = 0x2C; + static const HYPHEN = 0x2D; + static const PERIOD = 0x2E; + static const QUESTION = 0x3F; + static const COLON = 0x3A; + static const SEMICOLON = 0x3B; + static const EQUALS = 0x3D; + static const LEFT_SQUARE = 0x5B; + static const RIGHT_SQUARE = 0x5D; + static const LEFT_CURLY = 0x7B; + static const RIGHT_CURLY = 0x7D; + static const HASH = 0x23; + static const AMPERSAND = 0x26; + static const ASTERISK = 0x2A; + static const EXCLAMATION = 0x21; + static const VERTICAL_BAR = 0x7C; + static const LEFT_ANGLE = 0x3C; + static const RIGHT_ANGLE = 0x3E; + static const SINGLE_QUOTE = 0x27; + static const DOUBLE_QUOTE = 0x22; + static const PERCENT = 0x25; + static const AT = 0x40; + static const GRAVE_ACCENT = 0x60; + static const TILDE = 0x7E; + + static const NULL = 0x0; + static const BELL = 0x7; + static const BACKSPACE = 0x8; + static const VERTICAL_TAB = 0xB; + static const FORM_FEED = 0xC; + static const ESCAPE = 0x1B; + static const SLASH = 0x2F; + static const BACKSLASH = 0x5C; + static const UNDERSCORE = 0x5F; + static const NEL = 0x85; + static const NBSP = 0xA0; + static const LINE_SEPARATOR = 0x2028; + static const PARAGRAPH_SEPARATOR = 0x2029; + static const BOM = 0xFEFF; + + static const NUMBER_0 = 0x30; + static const NUMBER_9 = 0x39; + + static const LETTER_A = 0x61; + static const LETTER_B = 0x62; + static const LETTER_E = 0x65; + static const LETTER_F = 0x66; + static const LETTER_N = 0x6E; + static const LETTER_R = 0x72; + static const LETTER_T = 0x74; + static const LETTER_U = 0x75; + static const LETTER_V = 0x76; + static const LETTER_X = 0x78; + static const LETTER_Z = 0x7A; + + static const LETTER_CAP_A = 0x41; + static const LETTER_CAP_F = 0x46; + static const LETTER_CAP_L = 0x4C; + static const LETTER_CAP_N = 0x4E; + static const LETTER_CAP_P = 0x50; + static const LETTER_CAP_U = 0x55; + static const LETTER_CAP_X = 0x58; + static const LETTER_CAP_Z = 0x5A; + + /// The underlying [SpanScanner] used to read characters from the source text. + /// + /// This is also used to track line and column information and to generate + /// [SourceSpan]s. + final SpanScanner _scanner; + + /// Whether this scanner has produced a [TokenType.STREAM_START] token + /// indicating the beginning of the YAML stream. + var _streamStartProduced = false; + + /// Whether this scanner has produced a [TokenType.STREAM_END] token + /// indicating the end of the YAML stream. + var _streamEndProduced = false; + + /// The queue of tokens yet to be emitted. + /// + /// These are queued up in advance so that [TokenType.KEY] tokens can be + /// inserted once the scanner determines that a series of tokens represents a + /// mapping key. + final _tokens = new QueueList<Token>(); + + /// The number of tokens that have been emitted. + /// + /// This doesn't count tokens in [tokens]. + var _tokensParsed = 0; + + /// Whether the next token in [_tokens] is ready to be returned. + /// + /// It might not be ready if there may still be a [TokenType.KEY] inserted + /// before it. + var _tokenAvailable = false; + + /// The stack of indent levels for the current nested block contexts. + /// + /// The YAML spec specifies that the initial indentation level is -1 spaces. + final _indents = <int>[-1]; + + /// Whether a simple key is allowed in this context. + /// + /// A simple key refers to any mapping key that doesn't have an explicit "?". + var _simpleKeyAllowed = true; + + /// The stack of potential simple keys for each level of flow nesting. + /// + /// Entries in this list may be `null`, indicating that there is no valid + /// simple key for the associated level of nesting. + /// + /// When a ":" is parsed and there's a simple key available, a [TokenType.KEY] + /// token is inserted in [_tokens] before that key's token. This allows the + /// parser to tell that the key is intended to be a mapping key. + final _simpleKeys = <_SimpleKey>[null]; + + /// The current indentation level. + int get _indent => _indents.last; + + /// Whether the scanner's currently positioned in a block-level structure (as + /// opposed to flow-level). + bool get _inBlockContext => _simpleKeys.length == 1; + + /// Whether the current character is a line break or the end of the source. + bool get _isBreakOrEnd => _scanner.isDone || _isBreak; + + /// Whether the current character is a line break. + bool get _isBreak => _isBreakAt(0); + + /// Whether the current character is whitespace or the end of the source. + bool get _isBlankOrEnd => _isBlankOrEndAt(0); + + /// Whether the current character is whitespace. + bool get _isBlank => _isBlankAt(0); + + /// Whether the current character is a valid tag name character. + /// + /// See http://yaml.org/spec/1.2/spec.html#ns-tag-name. + bool get _isTagChar { + var char = _scanner.peekChar(); + if (char == null) return false; + switch (char) { + case HYPHEN: + case SEMICOLON: + case SLASH: + case COLON: + case AT: + case AMPERSAND: + case EQUALS: + case PLUS: + case DOLLAR: + case PERIOD: + case TILDE: + case QUESTION: + case ASTERISK: + case SINGLE_QUOTE: + case LEFT_PAREN: + case RIGHT_PAREN: + case PERCENT: + return true; + default: + return (char >= NUMBER_0 && char <= NUMBER_9) || + (char >= LETTER_A && char <= LETTER_Z) || + (char >= LETTER_CAP_A && char <= LETTER_CAP_Z); + } + } + + /// Whether the current character is a valid anchor name character. + /// + /// See http://yaml.org/spec/1.2/spec.html#ns-anchor-name. + bool get _isAnchorChar { + if (!_isNonSpace) return false; + + switch (_scanner.peekChar()) { + case COMMA: + case LEFT_SQUARE: + case RIGHT_SQUARE: + case LEFT_CURLY: + case RIGHT_CURLY: + return false; + default: + return true; + } + } + + /// Whether the character at the current position is a decimal digit. + bool get _isDigit { + var char = _scanner.peekChar(); + return char != null && (char >= NUMBER_0 && char <= NUMBER_9); + } + + /// Whether the character at the current position is a hexidecimal + /// digit. + bool get _isHex { + var char = _scanner.peekChar(); + if (char == null) return false; + return (char >= NUMBER_0 && char <= NUMBER_9) || + (char >= LETTER_A && char <= LETTER_F) || + (char >= LETTER_CAP_A && char <= LETTER_CAP_F); + } + + /// Whether the character at the current position is a plain character. + /// + /// See http://yaml.org/spec/1.2/spec.html#ns-plain-char(c). + bool get _isPlainChar => _isPlainCharAt(0); + + /// Whether the character at the current position is a printable character + /// other than a line break or byte-order mark. + /// + /// See http://yaml.org/spec/1.2/spec.html#nb-char. + bool get _isNonBreak { + var char = _scanner.peekChar(); + if (char == null) return false; + switch (char) { + case LF: + case CR: + case BOM: + return false; + case TAB: + case NEL: + return true; + default: + return (char >= 0x00020 && char <= 0x00007E) || + (char >= 0x000A0 && char <= 0x00D7FF) || + (char >= 0x0E000 && char <= 0x00FFFD) || + (char >= 0x10000 && char <= 0x10FFFF); + } + } + + /// Whether the character at the current position is a printable character + /// other than whitespace. + /// + /// See http://yaml.org/spec/1.2/spec.html#nb-char. + bool get _isNonSpace { + var char = _scanner.peekChar(); + if (char == null) return false; + switch (char) { + case LF: + case CR: + case BOM: + case SP: + return false; + case NEL: + return true; + default: + return (char >= 0x00020 && char <= 0x00007E) || + (char >= 0x000A0 && char <= 0x00D7FF) || + (char >= 0x0E000 && char <= 0x00FFFD) || + (char >= 0x10000 && char <= 0x10FFFF); + } + } + + /// Returns Whether or not the current character begins a documentation + /// indicator. + /// + /// If so, this sets the scanner's last match to that indicator. + bool get _isDocumentIndicator { + return _scanner.column == 0 && _isBlankOrEndAt(3) && + (_scanner.matches('---') || _scanner.matches('...')); + } + + /// Creates a scanner that scans [source]. + /// + /// [sourceUrl] can be a String or a [Uri]. + Scanner(String source, {sourceUrl}) + : _scanner = new SpanScanner(source, sourceUrl: sourceUrl); + + /// Consumes and returns the next token. + Token scan() { + if (_streamEndProduced) throw new StateError("Out of tokens."); + if (!_tokenAvailable) _fetchMoreTokens(); + + var token = _tokens.removeFirst(); + _tokenAvailable = false; + _tokensParsed++; + _streamEndProduced = token is Token && + token.type == TokenType.STREAM_END; + return token; + } + + /// Consumes the next token and returns the one after that. + Token advance() { + scan(); + return peek(); + } + + /// Returns the next token without consuming it. + Token peek() { + if (_streamEndProduced) return null; + if (!_tokenAvailable) _fetchMoreTokens(); + return _tokens.first; + } + + /// Ensures that [_tokens] contains at least one token which can be returned. + void _fetchMoreTokens() { + while (true) { + if (_tokens.isNotEmpty) { + _staleSimpleKeys(); + + // If the current token could be a simple key, we need to scan more + // tokens until we determine whether it is or not. Otherwise we might + // not emit the `KEY` token before we emit the value of the key. + if (!_simpleKeys.any((key) => + key != null && key.tokenNumber == _tokensParsed)) { + break; + } + } + + _fetchNextToken(); + } + _tokenAvailable = true; + } + + /// The dispatcher for token fetchers. + void _fetchNextToken() { + if (!_streamStartProduced) { + _fetchStreamStart(); + return; + } + + _scanToNextToken(); + _staleSimpleKeys(); + _unrollIndent(_scanner.column); + + if (_scanner.isDone) { + _fetchStreamEnd(); + return; + } + + if (_scanner.column == 0) { + if (_scanner.peekChar() == PERCENT) { + _fetchDirective(); + return; + } + + if (_isBlankOrEndAt(3)) { + if (_scanner.matches('---')) { + _fetchDocumentIndicator(TokenType.DOCUMENT_START); + return; + } + + if (_scanner.matches('...')) { + _fetchDocumentIndicator(TokenType.DOCUMENT_END); + return; + } + } + } + + switch (_scanner.peekChar()) { + case LEFT_SQUARE: + _fetchFlowCollectionStart(TokenType.FLOW_SEQUENCE_START); + return; + case LEFT_CURLY: + _fetchFlowCollectionStart(TokenType.FLOW_MAPPING_START); + return; + case RIGHT_SQUARE: + _fetchFlowCollectionEnd(TokenType.FLOW_SEQUENCE_END); + return; + case RIGHT_CURLY: + _fetchFlowCollectionEnd(TokenType.FLOW_MAPPING_END); + return; + case COMMA: + _fetchFlowEntry(); + return; + case ASTERISK: + _fetchAnchor(anchor: false); + return; + case AMPERSAND: + _fetchAnchor(anchor: true); + return; + case EXCLAMATION: + _fetchTag(); + return; + case SINGLE_QUOTE: + _fetchFlowScalar(singleQuote: true); + return; + case DOUBLE_QUOTE: + _fetchFlowScalar(singleQuote: false); + return; + case VERTICAL_BAR: + if (!_inBlockContext) _invalidScalarCharacter(); + _fetchBlockScalar(literal: true); + return; + case RIGHT_ANGLE: + if (!_inBlockContext) _invalidScalarCharacter(); + _fetchBlockScalar(literal: false); + return; + case PERCENT: + case AT: + case GRAVE_ACCENT: + _invalidScalarCharacter(); + return; + + // These characters may sometimes begin plain scalars. + case HYPHEN: + if (_isPlainCharAt(1)) { + _fetchPlainScalar(); + } else { + _fetchBlockEntry(); + } + return; + case QUESTION: + if (_isPlainCharAt(1)) { + _fetchPlainScalar(); + } else { + _fetchKey(); + } + return; + case COLON: + if (!_inBlockContext && _tokens.isNotEmpty) { + // If a colon follows a "JSON-like" value (an explicit map or list, or + // a quoted string) it isn't required to have whitespace after it + // since it unambiguously describes a map. + var token = _tokens.last; + if (token.type == TokenType.FLOW_SEQUENCE_END || + token.type == TokenType.FLOW_MAPPING_END || + (token.type == TokenType.SCALAR && token.style.isQuoted)) { + _fetchValue(); + return; + } + } + + if (_isPlainCharAt(1)) { + _fetchPlainScalar(); + } else { + _fetchValue(); + } + return; + default: + if (!_isNonBreak) _invalidScalarCharacter(); + + _fetchPlainScalar(); + return; + } + + throw 'Inaccessible'; + } + + /// Throws an error about a disallowed character. + void _invalidScalarCharacter() => + _scanner.error("Unexpected character.", length: 1); + + /// Checks the list of potential simple keys and remove the positions that + /// cannot contain simple keys anymore. + void _staleSimpleKeys() { + for (var i = 0; i < _simpleKeys.length; i++) { + var key = _simpleKeys[i]; + if (key == null) continue; + + // libyaml requires that all simple keys be a single line and no longer + // than 1024 characters. However, in section 7.4.2 of the spec + // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are + // only applied when the curly braces are omitted. It's difficult to + // retain enough context to know which keys need to have the restriction + // placed on them, so for now we go the other direction and allow + // everything but multiline simple keys in a block context. + if (!_inBlockContext) continue; + + if (key.location.line == _scanner.line) continue; + + if (key.required) { + throw new YamlException("Expected ':'.", _scanner.emptySpan); + } + + _simpleKeys[i] = null; + } + } + + /// Checks if a simple key may start at the current position and saves it if + /// so. + void _saveSimpleKey() { + // A simple key is required at the current position if the scanner is in the + // block context and the current column coincides with the indentation + // level. + var required = _inBlockContext && _indent == _scanner.column; + + // A simple key is required only when it is the first token in the current + // line. Therefore it is always allowed. But we add a check anyway. + assert(_simpleKeyAllowed || !required); + + if (!_simpleKeyAllowed) return; + + // If the current position may start a simple key, save it. + _removeSimpleKey(); + _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( + _tokensParsed + _tokens.length, + _scanner.location, + required: required); + } + + /// Removes a potential simple key at the current flow level. + void _removeSimpleKey() { + var key = _simpleKeys.last; + if (key != null && key.required) { + throw new YamlException("Could not find expected ':' for simple key.", + key.location.pointSpan()); + } + + _simpleKeys[_simpleKeys.length - 1] = null; + } + + /// Increases the flow level and resizes the simple key list. + void _increaseFlowLevel() { + _simpleKeys.add(null); + } + + /// Decreases the flow level. + void _decreaseFlowLevel() { + if (_inBlockContext) return; + _simpleKeys.removeLast(); + } + + /// Pushes the current indentation level to the stack and sets the new level + /// if [column] is greater than [_indent]. + /// + /// If it is, appends or inserts the specified token into [_tokens]. If + /// [tokenNumber] is provided, the corresponding token will be replaced; + /// otherwise, the token will be added at the end. + void _rollIndent(int column, TokenType type, SourceLocation location, + {int tokenNumber}) { + if (!_inBlockContext) return; + if (_indent != -1 && _indent >= column) return; + + // Push the current indentation level to the stack and set the new + // indentation level. + _indents.add(column); + + // Create a token and insert it into the queue. + var token = new Token(type, location.pointSpan()); + if (tokenNumber == null) { + _tokens.add(token); + } else { + _tokens.insert(tokenNumber - _tokensParsed, token); + } + } + + /// Pops indentation levels from [_indents] until the current level becomes + /// less than or equal to [column]. + /// + /// For each indentation level, appends a [TokenType.BLOCK_END] token. + void _unrollIndent(int column) { + if (!_inBlockContext) return; + + while (_indent > column) { + _tokens.add(new Token(TokenType.BLOCK_END, _scanner.emptySpan)); + _indents.removeLast(); + } + } + + /// Pops indentation levels from [_indents] until the current level resets to + /// -1. + /// + /// For each indentation level, appends a [TokenType.BLOCK_END] token. + void _resetIndent() => _unrollIndent(-1); + + /// Produces a [TokenType.STREAM_START] token. + void _fetchStreamStart() { + // Much of libyaml's initialization logic here is done in variable + // initializers instead. + _streamStartProduced = true; + _tokens.add(new Token(TokenType.STREAM_START, _scanner.emptySpan)); + } + + /// Produces a [TokenType.STREAM_END] token. + void _fetchStreamEnd() { + _resetIndent(); + _removeSimpleKey(); + _simpleKeyAllowed = false; + _tokens.add(new Token(TokenType.STREAM_END, _scanner.emptySpan)); + } + + /// Produces a [TokenType.VERSION_DIRECTIVE] or [TokenType.TAG_DIRECTIVE] + /// token. + void _fetchDirective() { + _resetIndent(); + _removeSimpleKey(); + _simpleKeyAllowed = false; + var directive = _scanDirective(); + if (directive != null) _tokens.add(directive); + } + + /// Produces a [TokenType.DOCUMENT_START] or [TokenType.DOCUMENT_END] token. + void _fetchDocumentIndicator(TokenType type) { + _resetIndent(); + _removeSimpleKey(); + _simpleKeyAllowed = false; + + // Consume the indicator token. + var start = _scanner.state; + _scanner.readChar(); + _scanner.readChar(); + _scanner.readChar(); + + _tokens.add(new Token(type, _scanner.spanFrom(start))); + } + + /// Produces a [TokenType.FLOW_SEQUENCE_START] or + /// [TokenType.FLOW_MAPPING_START] token. + void _fetchFlowCollectionStart(TokenType type) { + _saveSimpleKey(); + _increaseFlowLevel(); + _simpleKeyAllowed = true; + _addCharToken(type); + } + + /// Produces a [TokenType.FLOW_SEQUENCE_END] or [TokenType.FLOW_MAPPING_END] + /// token. + void _fetchFlowCollectionEnd(TokenType type) { + _removeSimpleKey(); + _decreaseFlowLevel(); + _simpleKeyAllowed = false; + _addCharToken(type); + } + + /// Produces a [TokenType.FLOW_ENTRY] token. + void _fetchFlowEntry() { + _removeSimpleKey(); + _simpleKeyAllowed = true; + _addCharToken(TokenType.FLOW_ENTRY); + } + + /// Produces a [TokenType.BLOCK_ENTRY] token. + void _fetchBlockEntry() { + if (_inBlockContext) { + if (!_simpleKeyAllowed) { + throw new YamlException( + "Block sequence entries are not allowed in this context.", + _scanner.emptySpan); + } + + _rollIndent( + _scanner.column, + TokenType.BLOCK_SEQUENCE_START, + _scanner.emptySpan.start); + } else { + // It is an error for the '-' indicator to occur in the flow context, but + // we let the Parser detect and report it because it's able to point to + // the context. + } + + _removeSimpleKey(); + _simpleKeyAllowed = true; + _addCharToken(TokenType.BLOCK_ENTRY); + } + + /// Produces the [TokenType.KEY] token. + void _fetchKey() { + if (_inBlockContext) { + if (!_simpleKeyAllowed) { + throw new YamlException("Mapping keys are not allowed in this context.", + _scanner.emptySpan); + } + + _rollIndent( + _scanner.column, + TokenType.BLOCK_MAPPING_START, + _scanner.emptySpan.start); + } + + // Simple keys are allowed after `?` in a block context. + _simpleKeyAllowed = _inBlockContext; + _addCharToken(TokenType.KEY); + } + + /// Produces the [TokenType.VALUE] token. + void _fetchValue() { + var simpleKey = _simpleKeys.last; + if (simpleKey != null) { + // Add a [TokenType.KEY] directive before the first token of the simple + // key so the parser knows that it's part of a key/value pair. + _tokens.insert(simpleKey.tokenNumber - _tokensParsed, + new Token(TokenType.KEY, simpleKey.location.pointSpan())); + + // In the block context, we may need to add the + // [TokenType.BLOCK_MAPPING_START] token. + _rollIndent( + simpleKey.location.column, + TokenType.BLOCK_MAPPING_START, + simpleKey.location, + tokenNumber: simpleKey.tokenNumber); + + // Remove the simple key. + _simpleKeys[_simpleKeys.length - 1] = null; + + // A simple key cannot follow another simple key. + _simpleKeyAllowed = false; + } else if (_inBlockContext) { + // If we're here, we've found the ':' indicator following a complex key. + + if (!_simpleKeyAllowed) { + throw new YamlException( + "Mapping values are not allowed in this context.", + _scanner.emptySpan); + } + + _rollIndent( + _scanner.column, + TokenType.BLOCK_MAPPING_START, + _scanner.location); + _simpleKeyAllowed = true; + } else if (_simpleKeyAllowed) { + // If we're here, we've found the ':' indicator with an empty key. This + // behavior differs from libyaml, which disallows empty implicit keys. + _simpleKeyAllowed = false; + _addCharToken(TokenType.KEY); + } + + _addCharToken(TokenType.VALUE); + } + + /// Adds a token with [type] to [_tokens]. + /// + /// The span of the new token is the current character. + void _addCharToken(TokenType type) { + var start = _scanner.state; + _scanner.readChar(); + _tokens.add(new Token(type, _scanner.spanFrom(start))); + } + + /// Produces a [TokenType.ALIAS] or [TokenType.ANCHOR] token. + void _fetchAnchor({bool anchor: true}) { + _saveSimpleKey(); + _simpleKeyAllowed = false; + _tokens.add(_scanAnchor(anchor: anchor)); + } + + /// Produces a [TokenType.TAG] token. + void _fetchTag() { + _saveSimpleKey(); + _simpleKeyAllowed = false; + _tokens.add(_scanTag()); + } + + /// Produces a [TokenType.SCALAR] token with style [ScalarStyle.LITERAL] or + /// [ScalarStyle.FOLDED]. + void _fetchBlockScalar({bool literal: false}) { + _removeSimpleKey(); + _simpleKeyAllowed = true; + _tokens.add(_scanBlockScalar(literal: literal)); + } + + /// Produces a [TokenType.SCALAR] token with style [ScalarStyle.SINGLE_QUOTED] + /// or [ScalarStyle.DOUBLE_QUOTED]. + void _fetchFlowScalar({bool singleQuote: false}) { + _saveSimpleKey(); + _simpleKeyAllowed = false; + _tokens.add(_scanFlowScalar(singleQuote: singleQuote)); + } + + /// Produces a [TokenType.SCALAR] token with style [ScalarStyle.PLAIN]. + void _fetchPlainScalar() { + _saveSimpleKey(); + _simpleKeyAllowed = false; + _tokens.add(_scanPlainScalar()); + } + + /// Eats whitespace and comments until the next token is found. + void _scanToNextToken() { + var afterLineBreak = false; + while (true) { + // Allow the BOM to start a line. + if (_scanner.column == 0) _scanner.scan("\uFEFF"); + + // Eat whitespace. + // + // libyaml disallows tabs after "-", "?", or ":", but the spec allows + // them. See section 6.2: http://yaml.org/spec/1.2/spec.html#id2778241. + while (_scanner.peekChar() == SP || + ((!_inBlockContext || !afterLineBreak) && + _scanner.peekChar() == TAB)) { + _scanner.readChar(); + } + + if (_scanner.peekChar() == TAB) { + _scanner.error("Tab characters are not allowed as indentation.", + length: 1); + } + + // Eat a comment until a line break. + _skipComment(); + + // If we're at a line break, eat it. + if (_isBreak) { + _skipLine(); + + // In the block context, a new line may start a simple key. + if (_inBlockContext) _simpleKeyAllowed = true; + afterLineBreak = true; + } else { + // Otherwise we've found a token. + break; + } + } + } + + /// Scans a [TokenType.YAML_DIRECTIVE] or [TokenType.TAG_DIRECTIVE] token. + /// + /// %YAML 1.2 # a comment \n + /// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + /// %TAG !yaml! tag:yaml.org,2002: \n + /// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Token _scanDirective() { + var start = _scanner.state; + + // Eat '%'. + _scanner.readChar(); + + var token; + var name = _scanDirectiveName(); + if (name == "YAML") { + token = _scanVersionDirectiveValue(start); + } else if (name == "TAG") { + token = _scanTagDirectiveValue(start); + } else { + warn("Warning: unknown directive.", _scanner.spanFrom(start)); + + // libyaml doesn't support unknown directives, but the spec says to ignore + // them and warn: http://yaml.org/spec/1.2/spec.html#id2781147. + while (!_isBreakOrEnd) { + _scanner.readChar(); + } + + return null; + } + + // Eat the rest of the line, including any comments. + _skipBlanks(); + _skipComment(); + + if (!_isBreakOrEnd) { + throw new YamlException( + "Expected comment or line break after directive.", + _scanner.spanFrom(start)); + } + + _skipLine(); + return token; + } + + /// Scans a directive name. + /// + /// %YAML 1.2 # a comment \n + /// ^^^^ + /// %TAG !yaml! tag:yaml.org,2002: \n + /// ^^^ + String _scanDirectiveName() { + // libyaml only allows word characters in directive names, but the spec + // disagrees: http://yaml.org/spec/1.2/spec.html#ns-directive-name. + var start = _scanner.position; + while (_isNonSpace) { + _scanner.readChar(); + } + + var name = _scanner.substring(start); + if (name.isEmpty) { + throw new YamlException("Expected directive name.", _scanner.emptySpan); + } else if (!_isBlankOrEnd) { + throw new YamlException( + "Unexpected character in directive name.", _scanner.emptySpan); + } + + return name; + } + + /// Scans the value of a version directive. + /// + /// %YAML 1.2 # a comment \n + /// ^^^^^^ + Token _scanVersionDirectiveValue(LineScannerState start) { + _skipBlanks(); + + var major = _scanVersionDirectiveNumber(); + _scanner.expect('.'); + var minor = _scanVersionDirectiveNumber(); + + return new VersionDirectiveToken(_scanner.spanFrom(start), major, minor); + } + + /// Scans the version number of a version directive. + /// + /// %YAML 1.2 # a comment \n + /// ^ + /// %YAML 1.2 # a comment \n + /// ^ + int _scanVersionDirectiveNumber() { + var start = _scanner.position; + while (_isDigit) { + _scanner.readChar(); + } + + var number = _scanner.substring(start); + if (number.isEmpty) { + throw new YamlException("Expected version number.", _scanner.emptySpan); + } + + return int.parse(number); + } + + /// Scans the value of a tag directive. + /// + /// %TAG !yaml! tag:yaml.org,2002: \n + /// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Token _scanTagDirectiveValue(LineScannerState start) { + _skipBlanks(); + + var handle = _scanTagHandle(directive: true); + if (!_isBlank) { + throw new YamlException("Expected whitespace.", _scanner.emptySpan); + } + + _skipBlanks(); + + var prefix = _scanTagUri(); + if (!_isBlankOrEnd) { + throw new YamlException("Expected whitespace.", _scanner.emptySpan); + } + + return new TagDirectiveToken(_scanner.spanFrom(start), handle, prefix); + } + + /// Scans a [TokenType.ANCHOR] token. + Token _scanAnchor({bool anchor: true}) { + var start = _scanner.state; + + // Eat the indicator character. + _scanner.readChar(); + + // libyaml only allows word characters in anchor names, but the spec + // disagrees: http://yaml.org/spec/1.2/spec.html#ns-anchor-char. + var startPosition = _scanner.position; + while (_isAnchorChar) { + _scanner.readChar(); + } + var name = _scanner.substring(startPosition); + + var next = _scanner.peekChar(); + if (name.isEmpty || + (!_isBlankOrEnd && next != QUESTION && next != COLON && + next != COMMA && next != RIGHT_SQUARE && next != RIGHT_CURLY && + next != PERCENT && next != AT && next != GRAVE_ACCENT)) { + throw new YamlException("Expected alphanumeric character.", + _scanner.emptySpan); + } + + if (anchor) { + return new AnchorToken(_scanner.spanFrom(start), name); + } else { + return new AliasToken(_scanner.spanFrom(start), name); + } + } + + /// Scans a [TokenType.TAG] token. + Token _scanTag() { + var handle; + var suffix; + var start = _scanner.state; + + // Check if the tag is in the canonical form. + if (_scanner.peekChar(1) == LEFT_ANGLE) { + // Eat '!<'. + _scanner.readChar(); + _scanner.readChar(); + + handle = ''; + suffix = _scanTagUri(); + + _scanner.expect('>'); + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + handle = _scanTagHandle(); + + if (handle.length > 1 && handle.startsWith('!') && handle.endsWith('!')) { + suffix = _scanTagUri(flowSeparators: false); + } else { + suffix = _scanTagUri(head: handle, flowSeparators: false); + + // There was no explicit handle. + if (suffix.isEmpty) { + // This is the special '!' tag. + handle = null; + suffix = '!'; + } else { + handle = '!'; + } + } + } + + // libyaml insists on whitespace after a tag, but example 7.2 indicates + // that it's not required: http://yaml.org/spec/1.2/spec.html#id2786720. + + return new TagToken(_scanner.spanFrom(start), handle, suffix); + } + + /// Scans a tag handle. + String _scanTagHandle({bool directive: false}) { + _scanner.expect('!'); + + var buffer = new StringBuffer('!'); + + // libyaml only allows word characters in tags, but the spec disagrees: + // http://yaml.org/spec/1.2/spec.html#ns-tag-char. + var start = _scanner.position; + while (_isTagChar) { + _scanner.readChar(); + } + buffer.write(_scanner.substring(start)); + + if (_scanner.peekChar() == EXCLAMATION) { + buffer.writeCharCode(_scanner.readChar()); + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be part of a + // URI. + if (directive && buffer.toString() != '!') _scanner.expect('!'); + } + + return buffer.toString(); + } + + /// Scans a tag URI. + /// + /// [head] is the initial portion of the tag that's already been scanned. + /// [flowSeparators] indicates whether the tag URI can contain flow + /// separators. + String _scanTagUri({String head, bool flowSeparators: true}) { + var length = head == null ? 0 : head.length; + var buffer = new StringBuffer(); + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if (length > 1) buffer.write(head.substring(1)); + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // + // In a shorthand tag annotation, the flow separators ',', '[', and ']' are + // disallowed. + var start = _scanner.position; + var char = _scanner.peekChar(); + while (_isTagChar || (flowSeparators && + (char == COMMA || char == LEFT_SQUARE || char == RIGHT_SQUARE))) { + _scanner.readChar(); + char = _scanner.peekChar(); + } + + // libyaml manually decodes the URL, but we don't have to do that. + return Uri.decodeFull(_scanner.substring(start)); + } + + /// Scans a block scalar. + Token _scanBlockScalar({bool literal: false}) { + var start = _scanner.state; + + // Eat the indicator '|' or '>'. + _scanner.readChar(); + + // Check for a chomping indicator. + var chomping = _Chomping.CLIP; + var increment = 0; + var char = _scanner.peekChar(); + if (char == PLUS || char == HYPHEN) { + chomping = char == PLUS ? _Chomping.KEEP : _Chomping.STRIP; + _scanner.readChar(); + + // Check for an indentation indicator. + if (_isDigit) { + // Check that the indentation is greater than 0. + if (_scanner.peekChar() == NUMBER_0) { + throw new YamlException( + "0 may not be used as an indentation indicator.", + _scanner.spanFrom(start)); + } + + increment = _scanner.readChar() - NUMBER_0; + } + } else if (_isDigit) { + // Do the same as above, but in the opposite order. + if (_scanner.peekChar() == NUMBER_0) { + throw new YamlException( + "0 may not be used as an indentation indicator.", + _scanner.spanFrom(start)); + } + + increment = _scanner.readChar() - NUMBER_0; + + char = _scanner.peekChar(); + if (char == PLUS || char == HYPHEN) { + chomping = char == PLUS ? _Chomping.KEEP : _Chomping.STRIP; + _scanner.readChar(); + } + } + + // Eat whitespace and comments to the end of the line. + _skipBlanks(); + _skipComment(); + + // Check if we're at the end of the line. + if (!_isBreakOrEnd) { + throw new YamlException("Expected comment or line break.", + _scanner.emptySpan); + } + + _skipLine(); + + // If the block scalar has an explicit indentation indicator, add that to + // the current indentation to get the indentation level for the scalar's + // contents. + var indent = 0; + if (increment != 0) { + indent = _indent >= 0 ? _indent + increment : increment; + } + + // Scan the leading line breaks to determine the indentation level if + // needed. + var pair = _scanBlockScalarBreaks(indent); + indent = pair.first; + var trailingBreaks = pair.last; + + // Scan the block scalar contents. + var buffer = new StringBuffer(); + var leadingBreak = ''; + var leadingBlank = false; + var trailingBlank = false; + while (_scanner.column == indent && !_scanner.isDone) { + // Check for a document indicator. libyaml doesn't do this, but the spec + // mandates it. See example 9.5: + // http://yaml.org/spec/1.2/spec.html#id2801606. + if (_isDocumentIndicator) break; + + // We are at the beginning of a non-empty line. + + // Is there trailing whitespace? + trailingBlank = _isBlank; + + // Check if we need to fold the leading line break. + if (!literal && leadingBreak.isNotEmpty && !leadingBlank && + !trailingBlank) { + // Do we need to join the lines with a space? + if (trailingBreaks.isEmpty) buffer.writeCharCode(SP); + } else { + buffer.write(leadingBreak); + } + leadingBreak = ''; + + // Append the remaining line breaks. + buffer.write(trailingBreaks); + + // Is there leading whitespace? + leadingBlank = _isBlank; + + var startPosition = _scanner.position; + while (!_isBreakOrEnd) { + _scanner.readChar(); + } + buffer.write(_scanner.substring(startPosition)); + + // libyaml always reads a line here, but this breaks on block scalars at + // the end of the document that end without newlines. See example 8.1: + // http://yaml.org/spec/1.2/spec.html#id2793888. + if (!_scanner.isDone) leadingBreak = _readLine(); + + // Eat the following indentation and spaces. + var pair = _scanBlockScalarBreaks(indent); + indent = pair.first; + trailingBreaks = pair.last; + } + + // Chomp the tail. + if (chomping != _Chomping.STRIP) buffer.write(leadingBreak); + if (chomping == _Chomping.KEEP) buffer.write(trailingBreaks); + + return new ScalarToken(_scanner.spanFrom(start), buffer.toString(), + literal ? ScalarStyle.LITERAL : ScalarStyle.FOLDED); + } + + /// Scans indentation spaces and line breaks for a block scalar. + /// + /// Determines the intendation level if needed. Returns the new indentation + /// level and the text of the line breaks. + Pair<int, String> _scanBlockScalarBreaks(int indent) { + var maxIndent = 0; + var breaks = new StringBuffer(); + + while (true) { + while ((indent == 0 || _scanner.column < indent) && + _scanner.peekChar() == SP) { + _scanner.readChar(); + } + + if (_scanner.column > maxIndent) maxIndent = _scanner.column; + + // libyaml throws an error here if a tab character is detected, but the + // spec treats tabs like any other non-space character. See example 8.2: + // http://yaml.org/spec/1.2/spec.html#id2794311. + + if (!_isBreak) break; + breaks.write(_readLine()); + } + + if (indent == 0) { + indent = maxIndent; + if (indent < _indent + 1) indent = _indent + 1; + + // libyaml forces indent to be at least 1 here, but that doesn't seem to + // be supported by the spec. + } + + return new Pair(indent, breaks.toString()); + } + + // Scans a quoted scalar. + Token _scanFlowScalar({bool singleQuote: false}) { + var start = _scanner.state; + var buffer = new StringBuffer(); + + // Eat the left quote. + _scanner.readChar(); + + while (true) { + // Check that there are no document indicators at the beginning of the + // line. + if (_isDocumentIndicator) { + _scanner.error("Unexpected document indicator."); + } + + if (_scanner.isDone) { + throw new YamlException("Unexpected end of file.", _scanner.emptySpan); + } + + var leadingBlanks = false; + while (!_isBlankOrEnd) { + var char = _scanner.peekChar(); + if (singleQuote && char == SINGLE_QUOTE && + _scanner.peekChar(1) == SINGLE_QUOTE) { + // An escaped single quote. + _scanner.readChar(); + _scanner.readChar(); + buffer.writeCharCode(SINGLE_QUOTE); + } else if (char == (singleQuote ? SINGLE_QUOTE : DOUBLE_QUOTE)) { + // The closing quote. + break; + } else if (!singleQuote && char == BACKSLASH && _isBreakAt(1)) { + // An escaped newline. + _scanner.readChar(); + _skipLine(); + leadingBlanks = true; + break; + } else if (!singleQuote && char == BACKSLASH) { + var escapeStart = _scanner.state; + + // An escape sequence. + var codeLength = null; + switch (_scanner.peekChar(1)) { + case NUMBER_0: + buffer.writeCharCode(NULL); + break; + case LETTER_A: + buffer.writeCharCode(BELL); + break; + case LETTER_B: + buffer.writeCharCode(BACKSPACE); + break; + case LETTER_T: + case TAB: + buffer.writeCharCode(TAB); + break; + case LETTER_N: + buffer.writeCharCode(LF); + break; + case LETTER_V: + buffer.writeCharCode(VERTICAL_TAB); + break; + case LETTER_F: + buffer.writeCharCode(FORM_FEED); + break; + case LETTER_R: + buffer.writeCharCode(CR); + break; + case LETTER_E: + buffer.writeCharCode(ESCAPE); + break; + case SP: + case DOUBLE_QUOTE: + case SLASH: + case BACKSLASH: + // libyaml doesn't support an escaped forward slash, but it was + // added in YAML 1.2. See section 5.7: + // http://yaml.org/spec/1.2/spec.html#id2776092 + buffer.writeCharCode(_scanner.peekChar(1)); + break; + case LETTER_CAP_N: + buffer.writeCharCode(NEL); + break; + case UNDERSCORE: + buffer.writeCharCode(NBSP); + break; + case LETTER_CAP_L: + buffer.writeCharCode(LINE_SEPARATOR); + break; + case LETTER_CAP_P: + buffer.writeCharCode(PARAGRAPH_SEPARATOR); + break; + case LETTER_X: + codeLength = 2; + break; + case LETTER_U: + codeLength = 4; + break; + case LETTER_CAP_U: + codeLength = 8; + break; + default: + throw new YamlException("Unknown escape character.", + _scanner.spanFrom(escapeStart)); + } + + _scanner.readChar(); + _scanner.readChar(); + + if (codeLength != null) { + var value = 0; + for (var i = 0; i < codeLength; i++) { + if (!_isHex) { + _scanner.readChar(); + throw new YamlException( + "Expected $codeLength-digit hexidecimal number.", + _scanner.spanFrom(escapeStart)); + } + + value = (value << 4) + _asHex(_scanner.readChar()); + } + + // Check the value and write the character. + if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF) { + throw new YamlException( + "Invalid Unicode character escape code.", + _scanner.spanFrom(escapeStart)); + } + + buffer.writeCharCode(value); + } + } else { + buffer.writeCharCode(_scanner.readChar()); + } + } + + // Check if we're at the end of a scalar. + if (_scanner.peekChar() == (singleQuote ? SINGLE_QUOTE : DOUBLE_QUOTE)) { + break; + } + + var whitespace = new StringBuffer(); + var leadingBreak = ''; + var trailingBreaks = new StringBuffer(); + while (_isBlank || _isBreak) { + if (_isBlank) { + // Consume a space or a tab. + if (!leadingBlanks) { + whitespace.writeCharCode(_scanner.readChar()); + } else { + _scanner.readChar(); + } + } else { + // Check if it's a first line break. + if (!leadingBlanks) { + whitespace.clear(); + leadingBreak = _readLine(); + leadingBlanks = true; + } else { + trailingBreaks.write(_readLine()); + } + } + } + + // Join the whitespace or fold line breaks. + if (leadingBlanks) { + if (leadingBreak.isNotEmpty && trailingBreaks.isEmpty) { + buffer.writeCharCode(SP); + } else { + buffer.write(trailingBreaks); + } + } else { + buffer.write(whitespace); + whitespace.clear(); + } + } + + // Eat the right quote. + _scanner.readChar(); + + return new ScalarToken(_scanner.spanFrom(start), buffer.toString(), + singleQuote ? ScalarStyle.SINGLE_QUOTED : ScalarStyle.DOUBLE_QUOTED); + } + + /// Scans a plain scalar. + Token _scanPlainScalar() { + var start = _scanner.state; + var buffer = new StringBuffer(); + var leadingBreak = ''; + var trailingBreaks = ''; + var whitespace = new StringBuffer(); + var indent = _indent + 1; + + while (true) { + // Check for a document indicator. + if (_isDocumentIndicator) break; + + // Check for a comment. + if (_scanner.peekChar() == HASH) break; + + if (_isPlainChar) { + // Join the whitespace or fold line breaks. + if (leadingBreak.isNotEmpty) { + if (trailingBreaks.isEmpty) { + buffer.writeCharCode(SP); + } else { + buffer.write(trailingBreaks); + } + leadingBreak = ''; + trailingBreaks = ''; + } else { + buffer.write(whitespace); + whitespace.clear(); + } + } + + // libyaml's notion of valid identifiers differs substantially from YAML + // 1.2's. We use [_isPlainChar] instead of libyaml's character here. + var startPosition = _scanner.position; + while (_isPlainChar) { + _scanner.readChar(); + } + buffer.write(_scanner.substring(startPosition)); + + // Is it the end? + if (!_isBlank && !_isBreak) break; + + while (_isBlank || _isBreak) { + if (_isBlank) { + // Check for a tab character messing up the intendation. + if (leadingBreak.isNotEmpty && _scanner.column < indent && + _scanner.peekChar() == TAB) { + _scanner.error("Expected a space but found a tab.", length: 1); + } + + if (leadingBreak.isEmpty) { + whitespace.writeCharCode(_scanner.readChar()); + } else { + _scanner.readChar(); + } + } else { + // Check if it's a first line break. + if (leadingBreak.isEmpty) { + leadingBreak = _readLine(); + whitespace.clear(); + } else { + trailingBreaks = _readLine(); + } + } + } + + // Check the indentation level. + if (_inBlockContext && _scanner.column < indent) break; + } + + // Allow a simple key after a plain scalar with leading blanks. + if (leadingBreak.isNotEmpty) _simpleKeyAllowed = true; + + return new ScalarToken(_scanner.spanFrom(start), buffer.toString(), + ScalarStyle.PLAIN); + } + + /// Moves past the current line break, if there is one. + void _skipLine() { + var char = _scanner.peekChar(); + if (char != CR && char != LF) return; + _scanner.readChar(); + if (char == CR && _scanner.peekChar() == LF) _scanner.readChar(); + } + + // Moves past the current line break and returns a newline. + String _readLine() { + var char = _scanner.peekChar(); + + // libyaml supports NEL, PS, and LS characters as line separators, but this + // is explicitly forbidden in section 5.4 of the YAML spec. + if (char != CR && char != LF) { + throw new YamlException("Expected newline.", _scanner.emptySpan); + } + + _scanner.readChar(); + // CR LF | CR | LF -> LF + if (char == CR && _scanner.peekChar() == LF) _scanner.readChar(); + return "\n"; + } + + // Returns whether the character at [offset] is whitespace. + bool _isBlankAt(int offset) { + var char = _scanner.peekChar(offset); + return char == SP || char == TAB; + } + + // Returns whether the character at [offset] is a line break. + bool _isBreakAt(int offset) { + // Libyaml considers NEL, LS, and PS to be line breaks as well, but that's + // contrary to the spec. + var char = _scanner.peekChar(offset); + return char == CR || char == LF; + } + + // Returns whether the character at [offset] is whitespace or past the end of + // the source. + bool _isBlankOrEndAt(int offset) { + var char = _scanner.peekChar(offset); + return char == null || char == SP || char == TAB || char == CR || + char == LF; + } + + /// Returns whether the character at [offset] is a plain character. + /// + /// See http://yaml.org/spec/1.2/spec.html#ns-plain-char(c). + bool _isPlainCharAt(int offset) { + switch (_scanner.peekChar(offset)) { + case COLON: + return _isPlainSafeAt(offset + 1); + case HASH: + var previous = _scanner.peekChar(offset - 1); + return previous != SP && previous != TAB; + default: + return _isPlainSafeAt(offset); + } + } + + /// Returns whether the character at [offset] is a plain-safe character. + /// + /// See http://yaml.org/spec/1.2/spec.html#ns-plain-safe(c). + bool _isPlainSafeAt(int offset) { + var char = _scanner.peekChar(offset); + switch (char) { + case COMMA: + case LEFT_SQUARE: + case RIGHT_SQUARE: + case LEFT_CURLY: + case RIGHT_CURLY: + // These characters are delimiters in a flow context and thus are only + // safe in a block context. + return _inBlockContext; + case SP: + case TAB: + case LF: + case CR: + case BOM: + return false; + case NEL: + return true; + default: + return char != null && + ((char >= 0x00020 && char <= 0x00007E) || + (char >= 0x000A0 && char <= 0x00D7FF) || + (char >= 0x0E000 && char <= 0x00FFFD) || + (char >= 0x10000 && char <= 0x10FFFF)); + } + } + + /// Returns the hexidecimal value of [char]. + int _asHex(int char) { + if (char <= NUMBER_9) return char - NUMBER_0; + if (char <= LETTER_CAP_F) return 10 + char - LETTER_CAP_A; + return 10 + char - LETTER_A; + } + + /// Moves the scanner past any blank characters. + void _skipBlanks() { + while (_isBlank) { + _scanner.readChar(); + } + } + + /// Moves the scanner past a comment, if one starts at the current position. + void _skipComment() { + if (_scanner.peekChar() != HASH) return; + while (!_isBreakOrEnd) { + _scanner.readChar(); + } + } +} + +/// A record of the location of a potential simple key. +class _SimpleKey { + /// The index of the token that begins the simple key. + /// + /// This is the index relative to all tokens emitted, rather than relative to + /// [_tokens]. + final int tokenNumber; + + /// The source location of the beginning of the simple key. + /// + /// This is used for error reporting and for determining when a simple key is + /// no longer on the current line. + final SourceLocation location; + + /// Whether this key must exist for the document to be scanned. + final bool required; + + _SimpleKey(this.tokenNumber, this.location, {bool required}) + : required = required; +} + +/// An enum of chomping indicators that describe how to handle trailing +/// whitespace for a block scalar. +/// +/// See http://yaml.org/spec/1.2/spec.html#id2794534. +class _Chomping { + /// All trailing whitespace is discarded. + static const STRIP = const _Chomping("STRIP"); + + /// A single trailing newline is retained. + static const CLIP = const _Chomping("CLIP"); + + /// All trailing whitespace is preserved. + static const KEEP = const _Chomping("KEEP"); + + final String name; + + const _Chomping(this.name); + + String toString() => name; +}
diff --git a/pkgs/yaml/lib/src/style.dart b/pkgs/yaml/lib/src/style.dart new file mode 100644 index 0000000..6305fce --- /dev/null +++ b/pkgs/yaml/lib/src/style.dart
@@ -0,0 +1,73 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library yaml.style; + +/// An enum of source scalar styles. +class ScalarStyle { + /// No source style was specified. + /// + /// This usually indicates a scalar constructed with [YamlScalar.wrap]. + static const ANY = const ScalarStyle._("ANY"); + + /// The plain scalar style, unquoted and without a prefix. + /// + /// See http://yaml.org/spec/1.2/spec.html#style/flow/plain. + static const PLAIN = const ScalarStyle._("PLAIN"); + + /// The literal scalar style, with a `|` prefix. + /// + /// See http://yaml.org/spec/1.2/spec.html#id2795688. + static const LITERAL = const ScalarStyle._("LITERAL"); + + + /// The folded scalar style, with a `>` prefix. + /// + /// See http://yaml.org/spec/1.2/spec.html#id2796251. + static const FOLDED = const ScalarStyle._("FOLDED"); + + /// The single-quoted scalar style. + /// + /// See http://yaml.org/spec/1.2/spec.html#style/flow/single-quoted. + static const SINGLE_QUOTED = const ScalarStyle._("SINGLE_QUOTED"); + + /// The double-quoted scalar style. + /// + /// See http://yaml.org/spec/1.2/spec.html#style/flow/double-quoted. + static const DOUBLE_QUOTED = const ScalarStyle._("DOUBLE_QUOTED"); + + final String name; + + /// Whether this is a quoted style ([SINGLE_QUOTED] or [DOUBLE_QUOTED]). + bool get isQuoted => this == SINGLE_QUOTED || this == DOUBLE_QUOTED; + + const ScalarStyle._(this.name); + + String toString() => name; +} + +/// An enum of collection styles. +class CollectionStyle { + /// No source style was specified. + /// + /// This usually indicates a collection constructed with [YamlList.wrap] or + /// [YamlMap.wrap]. + static const ANY = const CollectionStyle._("ANY"); + + /// The indentation-based block style. + /// + /// See http://yaml.org/spec/1.2/spec.html#id2797293. + static const BLOCK = const CollectionStyle._("BLOCK"); + + /// The delimiter-based block style. + /// + /// See http://yaml.org/spec/1.2/spec.html#id2790088. + static const FLOW = const CollectionStyle._("FLOW"); + + final String name; + + const CollectionStyle._(this.name); + + String toString() => name; +}
diff --git a/pkgs/yaml/lib/src/token.dart b/pkgs/yaml/lib/src/token.dart new file mode 100644 index 0000000..20ae547 --- /dev/null +++ b/pkgs/yaml/lib/src/token.dart
@@ -0,0 +1,148 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library yaml.token; + +import 'package:source_span/source_span.dart'; + +import 'style.dart'; + +/// A token emitted by a [Scanner]. +class Token { + /// The token type. + final TokenType type; + + /// The span associated with the token. + final FileSpan span; + + Token(this.type, this.span); + + String toString() => type.toString(); +} + +/// A token representing a `%YAML` directive. +class VersionDirectiveToken implements Token { + get type => TokenType.VERSION_DIRECTIVE; + final FileSpan span; + + /// The declared major version of the document. + final int major; + + /// The declared minor version of the document. + final int minor; + + VersionDirectiveToken(this.span, this.major, this.minor); + + String toString() => "VERSION_DIRECTIVE $major.$minor"; +} + +/// A token representing a `%TAG` directive. +class TagDirectiveToken implements Token { + get type => TokenType.TAG_DIRECTIVE; + final FileSpan span; + + /// The tag handle used in the document. + final String handle; + + /// The tag prefix that the handle maps to. + final String prefix; + + TagDirectiveToken(this.span, this.handle, this.prefix); + + String toString() => "TAG_DIRECTIVE $handle $prefix"; +} + +/// A token representing an anchor (`&foo`). +class AnchorToken implements Token { + get type => TokenType.ANCHOR; + final FileSpan span; + + /// The name of the anchor. + final String name; + + AnchorToken(this.span, this.name); + + String toString() => "ANCHOR $name"; +} + +/// A token representing an alias (`*foo`). +class AliasToken implements Token { + get type => TokenType.ALIAS; + final FileSpan span; + + /// The name of the anchor. + final String name; + + AliasToken(this.span, this.name); + + String toString() => "ALIAS $name"; +} + +/// A token representing a tag (`!foo`). +class TagToken implements Token { + get type => TokenType.TAG; + final FileSpan span; + + /// The tag handle. + final String handle; + + /// The tag suffix, or `null`. + final String suffix; + + TagToken(this.span, this.handle, this.suffix); + + String toString() => "TAG $handle $suffix"; +} + +/// A tkoen representing a scalar value. +class ScalarToken implements Token { + get type => TokenType.SCALAR; + final FileSpan span; + + /// The contents of the scalar. + final String value; + + /// The style of the scalar in the original source. + final ScalarStyle style; + + ScalarToken(this.span, this.value, this.style); + + String toString() => "SCALAR $style \"$value\""; +} + +/// An enum of types of [Token] object. +class TokenType { + static const STREAM_START = const TokenType._("STREAM_START"); + static const STREAM_END = const TokenType._("STREAM_END"); + + static const VERSION_DIRECTIVE = const TokenType._("VERSION_DIRECTIVE"); + static const TAG_DIRECTIVE = const TokenType._("TAG_DIRECTIVE"); + static const DOCUMENT_START = const TokenType._("DOCUMENT_START"); + static const DOCUMENT_END = const TokenType._("DOCUMENT_END"); + + static const BLOCK_SEQUENCE_START = const TokenType._("BLOCK_SEQUENCE_START"); + static const BLOCK_MAPPING_START = const TokenType._("BLOCK_MAPPING_START"); + static const BLOCK_END = const TokenType._("BLOCK_END"); + + static const FLOW_SEQUENCE_START = const TokenType._("FLOW_SEQUENCE_START"); + static const FLOW_SEQUENCE_END = const TokenType._("FLOW_SEQUENCE_END"); + static const FLOW_MAPPING_START = const TokenType._("FLOW_MAPPING_START"); + static const FLOW_MAPPING_END = const TokenType._("FLOW_MAPPING_END"); + + static const BLOCK_ENTRY = const TokenType._("BLOCK_ENTRY"); + static const FLOW_ENTRY = const TokenType._("FLOW_ENTRY"); + static const KEY = const TokenType._("KEY"); + static const VALUE = const TokenType._("VALUE"); + + static const ALIAS = const TokenType._("ALIAS"); + static const ANCHOR = const TokenType._("ANCHOR"); + static const TAG = const TokenType._("TAG"); + static const SCALAR = const TokenType._("SCALAR"); + + final String name; + + const TokenType._(this.name); + + String toString() => name; +}
diff --git a/pkgs/yaml/lib/src/utils.dart b/pkgs/yaml/lib/src/utils.dart index 84c1113..445221f 100644 --- a/pkgs/yaml/lib/src/utils.dart +++ b/pkgs/yaml/lib/src/utils.dart
@@ -4,6 +4,8 @@ library yaml.utils; +import 'package:source_span/source_span.dart'; + /// A pair of values. class Pair<E, F> { final E first; @@ -13,3 +15,30 @@ String toString() => '($first, $last)'; } + +/// Print a warning. +/// +/// If [span] is passed, associates the warning with that span. +void warn(String message, [SourceSpan span]) => + yamlWarningCallback(message, span); + +/// A callback for emitting a warning. +/// +/// [message] is the text of the warning. If [span] is passed, it's the portion +/// of the document that the warning is associated with and should be included +/// in the printed warning. +typedef YamlWarningCallback(String message, [SourceSpan span]); + +/// A callback for emitting a warning. +/// +/// In a very few cases, the YAML spec indicates that an implementation should +/// emit a warning. To do so, it calls this callback. The default implementation +/// prints a message using [print]. +YamlWarningCallback yamlWarningCallback = (message, [span]) { + // TODO(nweiz): Print to stderr with color when issue 6943 is fixed and + // dart:io is available. + if (span != null) message = span.message(message); + print(message); +}; + +
diff --git a/pkgs/yaml/lib/src/visitor.dart b/pkgs/yaml/lib/src/visitor.dart deleted file mode 100644 index 3f4c455..0000000 --- a/pkgs/yaml/lib/src/visitor.dart +++ /dev/null
@@ -1,30 +0,0 @@ -// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -library yaml.visitor; - -import 'equality.dart'; -import 'model.dart'; - -/// The visitor pattern for YAML documents. -class Visitor { - /// Returns [alias]. - visitAlias(AliasNode alias) => alias; - - /// Returns [scalar]. - visitScalar(ScalarNode scalar) => scalar; - - /// Visits each node in [seq] and returns a list of the results. - visitSequence(SequenceNode seq) - => seq.content.map((e) => e.visit(this)).toList(); - - /// Visits each key and value in [map] and returns a map of the results. - visitMapping(MappingNode map) { - var out = deepEqualsMap(); - for (var key in map.content.keys) { - out[key.visit(this)] = map.content[key].visit(this); - } - return out; - } -}
diff --git a/pkgs/yaml/lib/src/yaml_document.dart b/pkgs/yaml/lib/src/yaml_document.dart new file mode 100644 index 0000000..4c249a0 --- /dev/null +++ b/pkgs/yaml/lib/src/yaml_document.dart
@@ -0,0 +1,67 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library yaml.yaml_document; + +import 'dart:collection'; + +import 'package:source_span/source_span.dart'; + +import 'yaml_node.dart'; + +/// A YAML document, complete with metadata. +class YamlDocument { + /// The contents of the document. + final YamlNode contents; + + /// The span covering the entire document. + final SourceSpan span; + + /// The version directive for the document, if any. + final VersionDirective versionDirective; + + /// The tag directives for the document. + final List<TagDirective> tagDirectives; + + /// Whether the beginning of the document was implicit (versus explicit via + /// `===`). + final bool startImplicit; + + /// Whether the end of the document was implicit (versus explicit via `...`). + final bool endImplicit; + + /// Users of the library should not use this constructor. + YamlDocument.internal(this.contents, this.span, this.versionDirective, + List<TagDirective> tagDirectives, {this.startImplicit: false, + this.endImplicit: false}) + : tagDirectives = new UnmodifiableListView(tagDirectives); + + String toString() => contents.toString(); +} + +/// A directive indicating which version of YAML a document was written to. +class VersionDirective { + /// The major version number. + final int major; + + /// The minor version number. + final int minor; + + VersionDirective(this.major, this.minor); + + String toString() => "%YAML $major.$minor"; +} + +/// A directive describing a custom tag handle. +class TagDirective { + /// The handle for use in the document. + final String handle; + + /// The prefix that the handle maps to. + final String prefix; + + TagDirective(this.handle, this.prefix); + + String toString() => "%TAG $handle $prefix"; +}
diff --git a/pkgs/yaml/lib/src/yaml_node.dart b/pkgs/yaml/lib/src/yaml_node.dart index 6320156..027bc1b 100644 --- a/pkgs/yaml/lib/src/yaml_node.dart +++ b/pkgs/yaml/lib/src/yaml_node.dart
@@ -10,6 +10,7 @@ import 'package:source_span/source_span.dart'; import 'null_span.dart'; +import 'style.dart'; import 'yaml_node_wrapper.dart'; /// An interface for parsed nodes from a YAML source tree. @@ -26,7 +27,9 @@ /// /// [SourceSpan.message] can be used to produce a human-friendly message about /// this node. - SourceSpan get span; + SourceSpan get span => _span; + + SourceSpan _span; /// The inner value of this node. /// @@ -38,8 +41,6 @@ /// A read-only [Map] parsed from YAML. class YamlMap extends YamlNode with collection.MapMixin, UnmodifiableMapMixin { - final SourceSpan span; - /// A view of [this] where the keys and values are guaranteed to be /// [YamlNode]s. /// @@ -50,6 +51,9 @@ /// `dynamic` `map.nodes["foo"]` will still work. final Map<dynamic, YamlNode> nodes; + /// The style used for the map in the original document. + final CollectionStyle style; + Map get value => this; Iterable get keys => nodes.keys.map((node) => node.value); @@ -77,8 +81,10 @@ new YamlMapWrapper(dartMap, sourceUrl); /// Users of the library should not use this constructor. - YamlMap.internal(Map<dynamic, YamlNode> nodes, this.span) - : nodes = new UnmodifiableMapView<dynamic, YamlNode>(nodes); + YamlMap.internal(Map<dynamic, YamlNode> nodes, SourceSpan span, this.style) + : nodes = new UnmodifiableMapView<dynamic, YamlNode>(nodes) { + _span = span; + } operator [](key) { var node = nodes[key]; @@ -89,10 +95,11 @@ // TODO(nweiz): Use UnmodifiableListMixin when issue 18970 is fixed. /// A read-only [List] parsed from YAML. class YamlList extends YamlNode with collection.ListMixin { - final SourceSpan span; - final List<YamlNode> nodes; + /// The style used for the list in the original document. + final CollectionStyle style; + List get value => this; int get length => nodes.length; @@ -124,8 +131,10 @@ new YamlListWrapper(dartList, sourceUrl); /// Users of the library should not use this constructor. - YamlList.internal(List<YamlNode> nodes, this.span) - : nodes = new UnmodifiableListView<YamlNode>(nodes); + YamlList.internal(List<YamlNode> nodes, SourceSpan span, this.style) + : nodes = new UnmodifiableListView<YamlNode>(nodes) { + _span = span; + } operator [](int index) => nodes[index].value; @@ -136,10 +145,11 @@ /// A wrapped scalar value parsed from YAML. class YamlScalar extends YamlNode { - final SourceSpan span; - final value; + /// The style used for the scalar in the original document. + final ScalarStyle style; + /// Wraps a Dart value in a [YamlScalar]. /// /// This scalar's [span] won't have useful location information. However, it @@ -148,10 +158,21 @@ /// /// [sourceUrl] may be either a [String], a [Uri], or `null`. YamlScalar.wrap(this.value, {sourceUrl}) - : span = new NullSpan(sourceUrl); + : style = ScalarStyle.ANY { + _span = new NullSpan(sourceUrl); + } /// Users of the library should not use this constructor. - YamlScalar.internal(this.value, this.span); + YamlScalar.internal(this.value, SourceSpan span, this.style) { + _span = span; + } String toString() => value.toString(); } + +/// Sets the source span of a [YamlNode]. +/// +/// This method is not exposed publicly. +void setSpan(YamlNode node, SourceSpan span) { + node._span = span; +}
diff --git a/pkgs/yaml/lib/src/yaml_node_wrapper.dart b/pkgs/yaml/lib/src/yaml_node_wrapper.dart index d000dcb..be96ba4 100644 --- a/pkgs/yaml/lib/src/yaml_node_wrapper.dart +++ b/pkgs/yaml/lib/src/yaml_node_wrapper.dart
@@ -10,12 +10,15 @@ import 'package:source_span/source_span.dart'; import 'null_span.dart'; +import 'style.dart'; import 'yaml_node.dart'; /// A wrapper that makes a normal Dart map behave like a [YamlMap]. class YamlMapWrapper extends MapBase with pkg_collection.UnmodifiableMapMixin implements YamlMap { + final CollectionStyle style = CollectionStyle.ANY; + final Map _dartMap; final SourceSpan span; @@ -55,8 +58,8 @@ final SourceSpan _span; - Iterable get keys => - _dartMap.keys.map((key) => new YamlScalar.internal(key, _span)); + Iterable get keys => _dartMap.keys.map((key) => + new YamlScalar.internal(key, _span, ScalarStyle.ANY)); _YamlMapNodes(this._dartMap, this._span); @@ -76,6 +79,8 @@ // TODO(nweiz): Use UnmodifiableListMixin when issue 18970 is fixed. /// A wrapper that makes a normal Dart list behave like a [YamlList]. class YamlListWrapper extends ListBase implements YamlList { + final CollectionStyle style = CollectionStyle.ANY; + final List _dartList; final SourceSpan span; @@ -146,5 +151,5 @@ YamlNode _nodeForValue(value, SourceSpan span) { if (value is Map) return new YamlMapWrapper._(value, span); if (value is List) return new YamlListWrapper._(value, span); - return new YamlScalar.internal(value, span); + return new YamlScalar.internal(value, span, ScalarStyle.ANY); }
diff --git a/pkgs/yaml/lib/yaml.dart b/pkgs/yaml/lib/yaml.dart index e45dd6e..9af329a 100644 --- a/pkgs/yaml/lib/yaml.dart +++ b/pkgs/yaml/lib/yaml.dart
@@ -4,16 +4,17 @@ library yaml; -import 'package:string_scanner/string_scanner.dart'; - -import 'src/composer.dart'; -import 'src/constructor.dart'; -import 'src/parser.dart'; +import 'src/loader.dart'; +import 'src/style.dart'; +import 'src/yaml_document.dart'; import 'src/yaml_exception.dart'; import 'src/yaml_node.dart'; +export 'src/style.dart'; +export 'src/utils.dart' show YamlWarningCallback, yamlWarningCallback; +export 'src/yaml_document.dart'; export 'src/yaml_exception.dart'; -export 'src/yaml_node.dart'; +export 'src/yaml_node.dart' hide setSpan; /// Loads a single document from a YAML string. /// @@ -39,13 +40,29 @@ /// This is just like [loadYaml], except that where [loadYaml] would return a /// normal Dart value this returns a [YamlNode] instead. This allows the caller /// to be confident that the return value will always be a [YamlNode]. -YamlNode loadYamlNode(String yaml, {sourceUrl}) { - var stream = loadYamlStream(yaml, sourceUrl: sourceUrl); - if (stream.length != 1) { - throw new YamlException("Expected 1 document, were ${stream.length}.", - stream.span); +YamlNode loadYamlNode(String yaml, {sourceUrl}) => + loadYamlDocument(yaml, sourceUrl: sourceUrl).contents; + +/// Loads a single document from a YAML string as a [YamlDocument]. +/// +/// This is just like [loadYaml], except that where [loadYaml] would return a +/// normal Dart value this returns a [YamlDocument] instead. This allows the +/// caller to access document metadata. +YamlDocument loadYamlDocument(String yaml, {sourceUrl}) { + var loader = new Loader(yaml, sourceUrl: sourceUrl); + var document = loader.load(); + if (document == null) { + return new YamlDocument.internal( + new YamlScalar.internal(null, loader.span, ScalarStyle.ANY), + loader.span, null, const []); } - return stream.nodes[0]; + + var nextDocument = loader.load(); + if (nextDocument != null) { + throw new YamlException("Only expected one document.", nextDocument.span); + } + + return document; } /// Loads a stream of documents from a YAML string. @@ -62,15 +79,34 @@ /// If [sourceUrl] is passed, it's used as the URL from which the YAML /// originated for error reporting. It can be a [String], a [Uri], or `null`. YamlList loadYamlStream(String yaml, {sourceUrl}) { - var pair; - try { - pair = new Parser(yaml, sourceUrl).l_yamlStream(); - } on StringScannerException catch (error) { - throw new YamlException(error.message, error.span); + var loader = new Loader(yaml, sourceUrl: sourceUrl); + + var documents = []; + var document = loader.load(); + while (document != null) { + documents.add(document); + document = loader.load(); } - var nodes = pair.first - .map((doc) => new Constructor(new Composer(doc).compose()).construct()) - .toList(); - return new YamlList.internal(nodes, pair.last); + return new YamlList.internal( + documents.map((document) => document.contents).toList(), + loader.span, + CollectionStyle.ANY); +} + +/// Loads a stream of documents from a YAML string. +/// +/// This is like [loadYamlStream], except that it returns [YamlDocument]s with +/// metadata wrapping the document contents. +List<YamlDocument> loadYamlDocuments(String yaml, {sourceUrl}) { + var loader = new Loader(yaml, sourceUrl: sourceUrl); + + var documents = []; + var document = loader.load(); + while (document != null) { + documents.add(document); + document = loader.load(); + } + + return documents; }
diff --git a/pkgs/yaml/libyaml-license.txt b/pkgs/yaml/libyaml-license.txt new file mode 100644 index 0000000..050ced2 --- /dev/null +++ b/pkgs/yaml/libyaml-license.txt
@@ -0,0 +1,19 @@ +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
diff --git a/pkgs/yaml/pubspec.yaml b/pkgs/yaml/pubspec.yaml index 74194e5..d0d51c5 100644 --- a/pkgs/yaml/pubspec.yaml +++ b/pkgs/yaml/pubspec.yaml
@@ -1,12 +1,12 @@ name: yaml -version: 2.0.1+1 +version: 2.1.0 author: "Dart Team <misc@dartlang.org>" homepage: http://www.dartlang.org description: A parser for YAML. dependencies: - collection: ">=0.9.2 <2.0.0" + collection: ">=1.1.0 <2.0.0" path: ">=1.2.0 <2.0.0" - string_scanner: ">=0.1.0 <0.2.0" + string_scanner: ">=0.1.2 <0.2.0" source_span: ">=1.0.0 <2.0.0" dev_dependencies: unittest: ">=0.9.0 <0.12.0"
diff --git a/pkgs/yaml/test/yaml_test.dart b/pkgs/yaml/test/yaml_test.dart index 4fbb108..2d9a0e8 100644 --- a/pkgs/yaml/test/yaml_test.dart +++ b/pkgs/yaml/test/yaml_test.dart
@@ -4,7 +4,6 @@ library yaml.test; -// TODO(rnystrom): rewrite tests so that they don't need "Expect". import 'package:unittest/unittest.dart'; import 'package:yaml/yaml.dart'; @@ -16,7 +15,7 @@ group('has a friendly error message for', () { var tabError = predicate((e) => - e.toString().contains('tab characters are not allowed as indentation')); + e.toString().contains('Tab characters are not allowed as indentation')); test('using a tab as indentation', () { expect(() => loadYaml('foo:\n\tbar'), @@ -32,6 +31,29 @@ }); }); + group("refuses documents that declare version", () { + test("1.0", () { + expectYamlFails(""" + %YAML 1.0 + --- text + """); + }); + + test("1.3", () { + expectYamlFails(""" + %YAML 1.3 + --- text + """); + }); + + test("2.0", () { + expectYamlFails(""" + %YAML 2.0 + --- text + """); + }); + }); + // The following tests are all taken directly from the YAML spec // (http://www.yaml.org/spec/1.2/spec.html). Most of them are code examples // that are directly included in the spec, but additional tests are derived @@ -173,21 +195,21 @@ - Ken Griffey"""); }); - // test('[Example 2.10]', () { - // expectYamlLoads({ - // "hr": ["Mark McGwire", "Sammy Sosa"], - // "rbi": ["Sammy Sosa", "Ken Griffey"] - // }, - // """ - // --- - // hr: - // - Mark McGwire - // # Following node labeled SS - // - &SS Sammy Sosa - // rbi: - // - *SS # Subsequent occurrence - // - Ken Griffey"""); - // }); + test('[Example 2.10]', () { + expectYamlLoads({ + "hr": ["Mark McGwire", "Sammy Sosa"], + "rbi": ["Sammy Sosa", "Ken Griffey"] + }, + """ + --- + hr: + - Mark McGwire + # Following node labeled SS + - &SS Sammy Sosa + rbi: + - *SS # Subsequent occurrence + - Ken Griffey"""); + }); test('[Example 2.11]', () { var doc = deepEqualsMap(); @@ -536,12 +558,12 @@ """); }); - // test('[Example 5.9]', () { - // expectYamlLoads("text", - // """ - // %YAML 1.2 - // --- text"""); - // }); + test('[Example 5.9]', () { + expectYamlLoads("text", + """ + %YAML 1.2 + --- text"""); + }); test('[Example 5.10]', () { expectYamlFails("commercial-at: @text"); @@ -886,179 +908,179 @@ }); group('6.8: Directives', () { - // // TODO(nweiz): assert that this produces a warning - // test('[Example 6.13]', () { - // expectYamlLoads("foo", - // ''' - // %FOO bar baz # Should be ignored - // # with a warning. - // --- "foo"'''); - // }); + // TODO(nweiz): assert that this produces a warning + test('[Example 6.13]', () { + expectYamlLoads("foo", + ''' + %FOO bar baz # Should be ignored + # with a warning. + --- "foo"'''); + }); - // // TODO(nweiz): assert that this produces a warning - // test('[Example 6.14]', () { - // expectYamlLoads("foo", - // ''' - // %YAML 1.3 # Attempt parsing - // # with a warning - // --- - // "foo"'''); - // }); + // TODO(nweiz): assert that this produces a warning. + test('[Example 6.14]', () { + expectYamlLoads("foo", + ''' + %YAML 1.3 # Attempt parsing + # with a warning + --- + "foo"'''); + }); - // test('[Example 6.15]', () { - // expectYamlFails( - // """ - // %YAML 1.2 - // %YAML 1.1 - // foo"""); - // }); + test('[Example 6.15]', () { + expectYamlFails( + """ + %YAML 1.2 + %YAML 1.1 + foo"""); + }); - // test('[Example 6.16]', () { - // expectYamlLoads("foo", - // ''' - // %TAG !yaml! tag:yaml.org,2002: - // --- - // !yaml!str "foo"'''); - // }); + test('[Example 6.16]', () { + expectYamlLoads("foo", + ''' + %TAG !yaml! tag:yaml.org,2002: + --- + !yaml!str "foo"'''); + }); - // test('[Example 6.17]', () { - // ExpectYamlFails( - // """ - // %TAG ! !foo - // %TAG ! !foo - // bar"""); - // }); + test('[Example 6.17]', () { + expectYamlFails( + """ + %TAG ! !foo + %TAG ! !foo + bar"""); + }); // Examples 6.18 through 6.22 test custom tag URIs, which this // implementation currently doesn't plan to support. }); group('6.9: Node Properties', () { - // test('may be specified in any order', () { - // expectYamlLoads(["foo", "bar"], - // """ - // - !!str &a1 foo - // - &a2 !!str bar"""); - // }); + test('may be specified in any order', () { + expectYamlLoads(["foo", "bar"], + """ + - !!str &a1 foo + - &a2 !!str bar"""); + }); - // test('[Example 6.23]', () { - // expectYamlLoads({ - // "foo": "bar", - // "baz": "foo" - // }, - // ''' - // !!str &a1 "foo": - // !!str bar - // &a2 baz : *a1'''); - // }); + test('[Example 6.23]', () { + expectYamlLoads({ + "foo": "bar", + "baz": "foo" + }, + ''' + !!str &a1 "foo": + !!str bar + &a2 baz : *a1'''); + }); - // // Example 6.24 tests custom tag URIs, which this implementation currently - // // doesn't plan to support. + // Example 6.24 tests custom tag URIs, which this implementation currently + // doesn't plan to support. - // test('[Example 6.25]', () { - // expectYamlFails("- !<!> foo"); - // expectYamlFails("- !<\$:?> foo"); - // }); + test('[Example 6.25]', () { + expectYamlFails("- !<!> foo"); + expectYamlFails("- !<\$:?> foo"); + }); - // // Examples 6.26 and 6.27 test custom tag URIs, which this implementation - // // currently doesn't plan to support. + // Examples 6.26 and 6.27 test custom tag URIs, which this implementation + // currently doesn't plan to support. - // test('[Example 6.28]', () { - // expectYamlLoads(["12", 12, "12"], - // ''' - // # Assuming conventional resolution: - // - "12" - // - 12 - // - ! 12'''); - // }); + test('[Example 6.28]', () { + expectYamlLoads(["12", 12, "12"], + ''' + # Assuming conventional resolution: + - "12" + - 12 + - ! 12'''); + }); - // test('[Example 6.29]', () { - // expectYamlLoads({ - // "First occurrence": "Value", - // "Second occurrence": "anchor" - // }, - // """ - // First occurrence: &anchor Value - // Second occurrence: *anchor"""); - // }); + test('[Example 6.29]', () { + expectYamlLoads({ + "First occurrence": "Value", + "Second occurrence": "Value" + }, + """ + First occurrence: &anchor Value + Second occurrence: *anchor"""); + }); }); // Chapter 7: Flow Styles group('7.1: Alias Nodes', () { - // test("must not use an anchor that doesn't previously occur", () { - // expectYamlFails( - // """ - // - *anchor - // - &anchor foo"""); - // }); + test("must not use an anchor that doesn't previously occur", () { + expectYamlFails( + """ + - *anchor + - &anchor foo"""); + }); - // test("don't have to exist for a given anchor node", () { - // expectYamlLoads(["foo"], "- &anchor foo"); - // }); + test("don't have to exist for a given anchor node", () { + expectYamlLoads(["foo"], "- &anchor foo"); + }); - // group('must not specify', () { - // test('tag properties', () => expectYamlFails( - // """ - // - &anchor foo - // - !str *anchor"""); + group('must not specify', () { + test('tag properties', () => expectYamlFails( + """ + - &anchor foo + - !str *anchor""")); - // test('anchor properties', () => expectYamlFails( - // """ - // - &anchor foo - // - &anchor2 *anchor"""); + test('anchor properties', () => expectYamlFails( + """ + - &anchor foo + - &anchor2 *anchor""")); - // test('content', () => expectYamlFails( - // """ - // - &anchor foo - // - *anchor bar"""))); - // }); + test('content', () => expectYamlFails( + """ + - &anchor foo + - *anchor bar""")); + }); - // test('must preserve structural equality', () { - // var doc = loadYaml(cleanUpLiteral( - // """ - // anchor: &anchor [a, b, c] - // alias: *anchor"""); - // var anchorList = doc['anchor']; - // var aliasList = doc['alias']; - // expect(anchorList, same(aliasList)); + test('must preserve structural equality', () { + var doc = loadYaml(cleanUpLiteral( + """ + anchor: &anchor [a, b, c] + alias: *anchor""")); + var anchorList = doc['anchor']; + var aliasList = doc['alias']; + expect(anchorList, same(aliasList)); - // doc = loadYaml(cleanUpLiteral( - // """ - // ? &anchor [a, b, c] - // : ? *anchor - // : bar"""); - // anchorList = doc.keys[0]; - // aliasList = doc[['a', 'b', 'c']].keys[0]; - // expect(anchorList, same(aliasList)); - // }); + doc = loadYaml(cleanUpLiteral( + """ + ? &anchor [a, b, c] + : ? *anchor + : bar""")); + anchorList = doc.keys.first; + aliasList = doc[['a', 'b', 'c']].keys.first; + expect(anchorList, same(aliasList)); + }); - // test('[Example 7.1]', () { - // expectYamlLoads({ - // "First occurence": "Foo", - // "Second occurence": "Foo", - // "Override anchor": "Bar", - // "Reuse anchor": "Bar", - // }, - // """ - // First occurrence: &anchor Foo - // Second occurrence: *anchor - // Override anchor: &anchor Bar - // Reuse anchor: *anchor"""); - // }); + test('[Example 7.1]', () { + expectYamlLoads({ + "First occurrence": "Foo", + "Second occurrence": "Foo", + "Override anchor": "Bar", + "Reuse anchor": "Bar", + }, + """ + First occurrence: &anchor Foo + Second occurrence: *anchor + Override anchor: &anchor Bar + Reuse anchor: *anchor"""); + }); }); group('7.2: Empty Nodes', () { - // test('[Example 7.2]', () { - // expectYamlLoads({ - // "foo": "", - // "": "bar" - // }, - // """ - // { - // foo : !!str, - // !!str : bar, - // }"""); - // }); + test('[Example 7.2]', () { + expectYamlLoads({ + "foo": "", + "": "bar" + }, + """ + { + foo : !!str, + !!str : bar, + }"""); + }); test('[Example 7.3]', () { var doc = deepEqualsMap({"foo": null}); @@ -1284,17 +1306,18 @@ - [ {JSON: like}:adjacent ]"""); }); - test('[Example 7.22]', () { - expectYamlFails( - """ - [ foo - bar: invalid ]"""); - - // TODO(nweiz): enable this when we throw an error for long keys - // var dotList = new List.filled(1024, ' '); - // var dots = dotList.join(); - // expectYamlFails('[ "foo...$dots...bar": invalid ]'); - }); + // TODO(nweiz): enable this when we throw an error for long or multiline + // keys. + // test('[Example 7.22]', () { + // expectYamlFails( + // """ + // [ foo + // bar: invalid ]"""); + // + // var dotList = new List.filled(1024, ' '); + // var dots = dotList.join(); + // expectYamlFails('[ "foo...$dots...bar": invalid ]'); + // }); }); group('7.5: Flow Nodes', () { @@ -1308,15 +1331,15 @@ - c"""); }); - // test('[Example 7.24]', () { - // expectYamlLoads(["a", "b", "c", "c", ""], - // """ - // - !!str "a" - // - 'b' - // - &anchor "c" - // - *anchor - // - !!str"""); - // }); + test('[Example 7.24]', () { + expectYamlLoads(["a", "b", "c", "c", ""], + """ + - !!str "a" + - 'b' + - &anchor "c" + - *anchor + - !!str"""); + }); }); // Chapter 8: Block Styles @@ -1569,41 +1592,43 @@ : moon: white"""); }); - // test('[Example 8.20]', () { - // expectYamlLoads(["flow in block", "Block scalar\n", {"foo": "bar"}], - // ''' - // - - // "flow in block" - // - > - // Block scalar - // - !!map # Block collection - // foo : bar'''); - // }); + test('[Example 8.20]', () { + expectYamlLoads(["flow in block", "Block scalar\n", {"foo": "bar"}], + ''' + - + "flow in block" + - > + Block scalar + - !!map # Block collection + foo : bar'''); + }); - // test('[Example 8.21]', () { - // expectYamlLoads({"literal": "value", "folded": "value"}, - // """ - // literal: |2 - // value - // folded: - // !!str - // >1 - // value"""); - // }); + test('[Example 8.21]', () { + // The spec doesn't include a newline after "value" in the parsed map, but + // the block scalar is clipped so it should be retained. + expectYamlLoads({"literal": "value\n", "folded": "value"}, + """ + literal: |2 + value + folded: + !!str + >1 + value"""); + }); - // test('[Example 8.22]', () { - // expectYamlLoads({ - // "sequence": ["entry", ["nested"]], - // "mapping": {"foo": "bar"} - // }, - // """ - // sequence: !!seq - // - entry - // - !!seq - // - nested - // mapping: !!map - // foo: bar"""); - // }); + test('[Example 8.22]', () { + expectYamlLoads({ + "sequence": ["entry", ["nested"]], + "mapping": {"foo": "bar"} + }, + """ + sequence: !!seq + - entry + - !!seq + - nested + mapping: !!map + foo: bar"""); + }); }); // Chapter 9: YAML Character Stream @@ -1611,14 +1636,14 @@ // Example 9.1 tests the use of a BOM, which this implementation currently // doesn't plan to support. - // test('[Example 9.2]', () { - // expectYamlLoads("Document", - // """ - // %YAML 1.2 - // --- - // Document - // ... # Suffix"""); - // }); + test('[Example 9.2]', () { + expectYamlLoads("Document", + """ + %YAML 1.2 + --- + Document + ... # Suffix"""); + }); test('[Example 9.3]', () { // The spec example indicates that the comment after "%!PS-Adobe-2.0" @@ -1653,81 +1678,83 @@ ..."""); }); - // test('[Example 9.5]', () { - // expectYamlStreamLoads(["%!PS-Adobe-2.0\n", null], - // """ - // %YAML 1.2 - // --- | - // %!PS-Adobe-2.0 - // ... - // %YAML1.2 - // --- - // # Empty - // ..."""); - // }); + test('[Example 9.5]', () { + // The spec doesn't have a space between the second + // "YAML" and "1.2", but this seems to be a typo. + expectYamlStreamLoads(["%!PS-Adobe-2.0\n", null], + """ + %YAML 1.2 + --- | + %!PS-Adobe-2.0 + ... + %YAML 1.2 + --- + # Empty + ..."""); + }); - // test('[Example 9.6]', () { - // expectYamlStreamLoads(["Document", null, {"matches %": 20}], - // """ - // Document - // --- - // # Empty - // ... - // %YAML 1.2 - // --- - // matches %: 20"""); - // }); + test('[Example 9.6]', () { + expectYamlStreamLoads(["Document", null, {"matches %": 20}], + """ + Document + --- + # Empty + ... + %YAML 1.2 + --- + matches %: 20"""); + }); }); // Chapter 10: Recommended Schemas group('10.1: Failsafe Schema', () { - // test('[Example 10.1]', () { - // expectYamlStreamLoads({ - // "Block style": { - // "Clark": "Evans", - // "Ingy": "döt Net", - // "Oren": "Ben-Kiki" - // }, - // "Flow style": { - // "Clark": "Evans", - // "Ingy": "döt Net", - // "Oren": "Ben-Kiki" - // } - // }, - // """ - // Block style: !!map - // Clark : Evans - // Ingy : döt Net - // Oren : Ben-Kiki + test('[Example 10.1]', () { + expectYamlLoads({ + "Block style": { + "Clark": "Evans", + "Ingy": "döt Net", + "Oren": "Ben-Kiki" + }, + "Flow style": { + "Clark": "Evans", + "Ingy": "döt Net", + "Oren": "Ben-Kiki" + } + }, + """ + Block style: !!map + Clark : Evans + Ingy : döt Net + Oren : Ben-Kiki - // Flow style: !!map { Clark: Evans, Ingy: döt Net, Oren: Ben-Kiki }"""); - // }); + Flow style: !!map { Clark: Evans, Ingy: döt Net, Oren: Ben-Kiki }"""); + }); - // test('[Example 10.2]', () { - // expectYamlStreamLoads({ - // "Block style": ["Clark Evans", "Ingy döt Net", "Oren Ben-Kiki"], - // "Flow style": ["Clark Evans", "Ingy döt Net", "Oren Ben-Kiki"] - // }, - // """ - // Block style: !!seq - // - Clark Evans - // - Ingy döt Net - // - Oren Ben-Kiki + test('[Example 10.2]', () { + expectYamlLoads({ + "Block style": ["Clark Evans", "Ingy döt Net", "Oren Ben-Kiki"], + "Flow style": ["Clark Evans", "Ingy döt Net", "Oren Ben-Kiki"] + }, + """ + Block style: !!seq + - Clark Evans + - Ingy döt Net + - Oren Ben-Kiki - // Flow style: !!seq [ Clark Evans, Ingy döt Net, Oren Ben-Kiki ]"""); - // }); + Flow style: !!seq [ Clark Evans, Ingy döt Net, Oren Ben-Kiki ]"""); + }); - // test('[Example 10.3]', () { - // expectYamlStreamLoads({ - // "Block style": "String: just a theory.", - // "Flow style": "String: just a theory." - // }, - // ''' - // Block style: !!str |- - // String: just a theory. + test('[Example 10.3]', () { + expectYamlLoads({ + "Block style": "String: just a theory.", + "Flow style": "String: just a theory." + }, + ''' + Block style: !!str |- + String: just a theory. - // Flow style: !!str "String: just a theory."'''); - // }); + Flow style: !!str "String: just a theory."'''); + }); }); group('10.2: JSON Schema', () {