Rework the parser to parse code fragments rather than value generators

What this means is that data file authors can specify values that are
extracted from the source being updated by writing a single string (in a
format that's a very limited subset of Dart) rather than by writing a
potentially deeply nested map structure.

Change-Id: I06011983310b1c5a4e921c3f762f5b39818ced17
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/169148
Reviewed-by: Phil Quitslund <pquitslund@google.com>
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>
diff --git a/pkg/analysis_server/lib/src/services/correction/fix/data_driven/code_fragment_parser.dart b/pkg/analysis_server/lib/src/services/correction/fix/data_driven/code_fragment_parser.dart
new file mode 100644
index 0000000..773eab0
--- /dev/null
+++ b/pkg/analysis_server/lib/src/services/correction/fix/data_driven/code_fragment_parser.dart
@@ -0,0 +1,311 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/src/services/correction/fix/data_driven/accessor.dart';
+import 'package:analysis_server/src/services/correction/fix/data_driven/parameter_reference.dart';
+import 'package:analysis_server/src/services/correction/fix/data_driven/transform_set_error_code.dart';
+import 'package:analyzer/error/listener.dart';
+
+/// A parser for the textual representation of a code fragment.
+class CodeFragmentParser {
+  /// The error reporter to which diagnostics will be reported.
+  final ErrorReporter errorReporter;
+
+  /// The amount to be added to translate from offsets within the content to
+  /// offsets within the file.
+  int delta;
+
+  /// The tokens being parsed.
+  List<_Token> tokens;
+
+  /// The accessors that have been parsed.
+  List<Accessor> accessors = [];
+
+  /// Initialize a newly created parser to report errors to the [errorReporter].
+  CodeFragmentParser(this.errorReporter);
+
+  /// Parse the [content] into a list of accessors. Add the [delta] to translate
+  /// from offsets within the content to offsets within the file.
+  ///
+  /// <content> ::=
+  ///   <accessor> ('.' <accessor>)*
+  List<Accessor> parse(String content, int delta) {
+    this.delta = delta;
+    tokens = _CodeFragmentScanner(content, delta, errorReporter).scan();
+    if (tokens == null) {
+      // The error has already been reported.
+      return null;
+    }
+    var index = _parseAccessor(0);
+    while (index < tokens.length) {
+      var token = tokens[index];
+      if (token.kind == _TokenKind.period) {
+        index = _parseAccessor(index + 1);
+      } else {
+        errorReporter.reportErrorForOffset(TransformSetErrorCode.wrongToken,
+            token.offset + delta, token.length, ['.', token.kind.displayName]);
+        return null;
+      }
+    }
+    return accessors;
+  }
+
+  /// Return the token at the given [index] if it exists and if it has one of
+  /// the [validKinds]. Report an error and return `null` if those conditions
+  /// aren't met.
+  _Token _expect(int index, List<_TokenKind> validKinds) {
+    String validKindsDisplayString() {
+      var buffer = StringBuffer();
+      for (var i = 0; i < validKinds.length; i++) {
+        if (i > 0) {
+          if (i == validKinds.length - 1) {
+            buffer.write(' or ');
+          } else {
+            buffer.write(', ');
+          }
+        }
+        buffer.write(validKinds[i].displayName);
+      }
+      return buffer.toString();
+    }
+
+    if (index >= tokens.length) {
+      var offset = 0;
+      var length = 0;
+      if (tokens.isNotEmpty) {
+        var last = tokens.last;
+        offset = last.offset;
+        length = last.length;
+      }
+      errorReporter.reportErrorForOffset(TransformSetErrorCode.missingToken,
+          offset + delta, length, [validKindsDisplayString()]);
+      return null;
+    }
+    var token = tokens[index];
+    if (!validKinds.contains(token.kind)) {
+      errorReporter.reportErrorForOffset(
+          TransformSetErrorCode.wrongToken,
+          token.offset + delta,
+          token.length,
+          [validKindsDisplayString(), token.kind.displayName]);
+      return null;
+    }
+    return token;
+  }
+
+  /// Parse an accessor.
+  ///
+  /// <accessor> ::=
+  ///   <identifier> '[' (<integer> | <identifier>) ']'
+  int _parseAccessor(int index) {
+    var token = _expect(index, const [_TokenKind.identifier]);
+    if (token == null) {
+      // The error has already been reported.
+      return tokens.length;
+    }
+    var identifier = token.lexeme;
+    if (identifier == 'arguments') {
+      token = _expect(index + 1, const [_TokenKind.openSquareBracket]);
+      if (token == null) {
+        // The error has already been reported.
+        return tokens.length;
+      }
+      token = _expect(index + 2, [_TokenKind.identifier, _TokenKind.integer]);
+      if (token == null) {
+        // The error has already been reported.
+        return tokens.length;
+      }
+      ParameterReference reference;
+      if (token.kind == _TokenKind.identifier) {
+        reference = NamedParameterReference(token.lexeme);
+      } else {
+        var argumentIndex = int.parse(token.lexeme);
+        reference = PositionalParameterReference(argumentIndex);
+      }
+      token = _expect(index + 3, [_TokenKind.closeSquareBracket]);
+      if (token == null) {
+        // The error has already been reported.
+        return tokens.length;
+      }
+      accessors.add(ArgumentAccessor(reference));
+      return index + 4;
+    } else if (identifier == 'typeArguments') {
+      token = _expect(index + 1, const [_TokenKind.openSquareBracket]);
+      if (token == null) {
+        // The error has already been reported.
+        return tokens.length;
+      }
+      token = _expect(index + 2, [_TokenKind.integer]);
+      if (token == null) {
+        // The error has already been reported.
+        return tokens.length;
+      }
+      var argumentIndex = int.parse(token.lexeme);
+      var reference = PositionalParameterReference(argumentIndex);
+      token = _expect(index + 3, [_TokenKind.closeSquareBracket]);
+      if (token == null) {
+        // The error has already been reported.
+        return tokens.length;
+      }
+      accessors.add(ArgumentAccessor(reference));
+      return index + 4;
+    } else {
+      errorReporter.reportErrorForOffset(TransformSetErrorCode.unknownAccessor,
+          token.offset, token.length, [identifier]);
+      return tokens.length;
+    }
+  }
+}
+
+/// A scanner for the textual representation of a code fragment.
+class _CodeFragmentScanner {
+  static final int $0 = '0'.codeUnitAt(0);
+
+  static final int $9 = '9'.codeUnitAt(0);
+
+  static final int $a = 'a'.codeUnitAt(0);
+
+  static final int $z = 'z'.codeUnitAt(0);
+
+  static final int $A = 'A'.codeUnitAt(0);
+  static final int $Z = 'Z'.codeUnitAt(0);
+  static final int closeSquareBracket = ']'.codeUnitAt(0);
+  static final int carriageReturn = '\r'.codeUnitAt(0);
+  static final int newline = '\n'.codeUnitAt(0);
+  static final int openSquareBracket = '['.codeUnitAt(0);
+  static final int period = '.'.codeUnitAt(0);
+  static final int space = ' '.codeUnitAt(0);
+
+  /// The string being scanned.
+  final String content;
+
+  /// The length of the string being scanned.
+  final int length;
+
+  /// The offset in the file of the first character in the string being scanned.
+  final int delta;
+
+  /// The error reporter to which diagnostics will be reported.
+  final ErrorReporter errorReporter;
+
+  /// Initialize a newly created scanner to scan the given [content].
+  _CodeFragmentScanner(this.content, this.delta, this.errorReporter)
+      : length = content.length;
+
+  /// Return the tokens in the content, or `null` if there is an error in the
+  /// content that prevents it from being scanned.
+  List<_Token> scan() {
+    if (content.isEmpty) {}
+    var length = content.length;
+    var offset = _skipWhitespace(0);
+    var tokens = <_Token>[];
+    while (offset < length) {
+      var char = content.codeUnitAt(offset);
+      if (char == closeSquareBracket) {
+        tokens.add(_Token(offset, _TokenKind.closeSquareBracket, ']'));
+        offset++;
+      } else if (char == openSquareBracket) {
+        tokens.add(_Token(offset, _TokenKind.openSquareBracket, '['));
+        offset++;
+      } else if (char == period) {
+        tokens.add(_Token(offset, _TokenKind.period, '.'));
+        offset++;
+      } else if (_isLetter(char)) {
+        var start = offset;
+        offset++;
+        while (offset < length && _isLetter(content.codeUnitAt(offset))) {
+          offset++;
+        }
+        tokens.add(_Token(
+            start, _TokenKind.identifier, content.substring(start, offset)));
+      } else if (_isDigit(char)) {
+        var start = offset;
+        offset++;
+        while (offset < length && _isDigit(content.codeUnitAt(offset))) {
+          offset++;
+        }
+        tokens.add(_Token(
+            start, _TokenKind.integer, content.substring(start, offset)));
+      } else {
+        errorReporter.reportErrorForOffset(
+            TransformSetErrorCode.invalidCharacter,
+            offset + delta,
+            1,
+            [content.substring(offset, offset + 1)]);
+        return null;
+      }
+      offset = _skipWhitespace(offset);
+    }
+    return tokens;
+  }
+
+  /// Return `true` if the [char] is a digit.
+  bool _isDigit(int char) => (char >= $0 && char <= $9);
+
+  /// Return `true` if the [char] is a letter.
+  bool _isLetter(int char) =>
+      (char >= $a && char <= $z) || (char >= $A && char <= $Z);
+
+  /// Return `true` if the [char] is a whitespace character.
+  bool _isWhitespace(int char) =>
+      char == space || char == newline || char == carriageReturn;
+
+  /// Return the index of the first character at or after the given [offset]
+  /// that isn't a whitespace character.
+  int _skipWhitespace(int offset) {
+    while (offset < length) {
+      var char = content.codeUnitAt(offset);
+      if (!_isWhitespace(char)) {
+        return offset;
+      }
+      offset++;
+    }
+    return offset;
+  }
+}
+
+/// A token in a code fragment's string representation.
+class _Token {
+  /// The offset of the token.
+  final int offset;
+
+  /// The kind of the token.
+  final _TokenKind kind;
+
+  /// The lexeme of the token.
+  final String lexeme;
+
+  /// Initialize a newly created token.
+  _Token(this.offset, this.kind, this.lexeme);
+
+  /// Return the length of this token.
+  int get length => lexeme.length;
+}
+
+/// An indication of the kind of a token.
+enum _TokenKind {
+  closeSquareBracket,
+  identifier,
+  integer,
+  openSquareBracket,
+  period,
+}
+
+extension on _TokenKind {
+  String get displayName {
+    switch (this) {
+      case _TokenKind.closeSquareBracket:
+        return "']'";
+      case _TokenKind.identifier:
+        return 'an identifier';
+      case _TokenKind.integer:
+        return 'an integer';
+      case _TokenKind.openSquareBracket:
+        return "'['";
+      case _TokenKind.period:
+        return "'.'";
+    }
+    return '';
+  }
+}
diff --git a/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_error_code.dart b/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_error_code.dart
index 8394acd..2788422 100644
--- a/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_error_code.dart
+++ b/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_error_code.dart
@@ -10,6 +10,13 @@
 class TransformSetErrorCode extends ErrorCode {
   /**
    * Parameters:
+   * 0: the character that is invalid
+   */
+  static const TransformSetErrorCode invalidCharacter =
+      TransformSetErrorCode('invalid_character', "Invalid character '{0}'.");
+
+  /**
+   * Parameters:
    * 0: the key with which the value is associated
    * 1: the expected type of the value
    * 0: the actual type of the value
@@ -42,6 +49,13 @@
 
   /**
    * Parameters:
+   * 0: a description of the expected kinds of tokens
+   */
+  static const TransformSetErrorCode missingToken =
+      TransformSetErrorCode('missing_token', "Expected to find {0}.");
+
+  /**
+   * Parameters:
    * 0: the missing key
    */
   static const TransformSetErrorCode undefinedVariable = TransformSetErrorCode(
@@ -49,6 +63,13 @@
 
   /**
    * Parameters:
+   * 0: a description of the expected kind of token
+   */
+  static const TransformSetErrorCode unknownAccessor = TransformSetErrorCode(
+      'unknown_accessor', "The accessor '{0}' is invalid.");
+
+  /**
+   * Parameters:
    * 0: the unsupported key
    */
   static const TransformSetErrorCode unsupportedKey = TransformSetErrorCode(
@@ -56,6 +77,14 @@
 
   /**
    * Parameters:
+   * 0: a description of the expected kind of token
+   * 1: a description of the actial kind of token
+   */
+  static const TransformSetErrorCode wrongToken = TransformSetErrorCode(
+      'wrong_token', "Expected to find {0}, but found {1}.");
+
+  /**
+   * Parameters:
    * 0: the message produced by the YAML parser
    */
   static const TransformSetErrorCode yamlSyntaxError =
diff --git a/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_parser.dart b/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_parser.dart
index 86ea95b..bad163e 100644
--- a/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_parser.dart
+++ b/pkg/analysis_server/lib/src/services/correction/fix/data_driven/transform_set_parser.dart
@@ -2,9 +2,9 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'package:analysis_server/src/services/correction/fix/data_driven/accessor.dart';
 import 'package:analysis_server/src/services/correction/fix/data_driven/add_type_parameter.dart';
 import 'package:analysis_server/src/services/correction/fix/data_driven/change.dart';
+import 'package:analysis_server/src/services/correction/fix/data_driven/code_fragment_parser.dart';
 import 'package:analysis_server/src/services/correction/fix/data_driven/code_template.dart';
 import 'package:analysis_server/src/services/correction/fix/data_driven/element_descriptor.dart';
 import 'package:analysis_server/src/services/correction/fix/data_driven/element_kind.dart';
@@ -68,6 +68,7 @@
   static const String _transformsKey = 'transforms';
   static const String _typedefKey = 'typedef';
   static const String _urisKey = 'uris';
+  static const String _valueKey = 'value';
   static const String _variableKey = 'variable';
   static const String _variablesKey = 'variables';
   static const String _versionKey = 'version';
@@ -85,7 +86,7 @@
 
   static const String _addParameterKind = 'addParameter';
   static const String _addTypeParameterKind = 'addTypeParameter';
-  static const String _argumentKind = 'argument';
+  static const String _fragmentKind = 'fragment';
   static const String _importKind = 'import';
   static const String _removeParameterKind = 'removeParameter';
   static const String _renameKind = 'rename';
@@ -192,6 +193,15 @@
     return node.runtimeType.toString();
   }
 
+  /// Return the offset of the first character in the [string], exclusive of any
+  /// surrounding quotes.
+  int _offsetOfString(YamlScalar string) {
+    // TODO(brianwilkerson) We add 1 to account for the quotes around the
+    //  string, but quotes aren't required, so we need to use the style of the
+    //  [string] is to get the right offset.
+    return string.span.start.offset + 1;
+  }
+
   /// Return the result of parsing the file [content] into a YAML node.
   YamlNode _parseYaml(String content) {
     try {
@@ -349,37 +359,6 @@
         argumentValue: argumentValue);
   }
 
-  /// Translate the [node] into a value extractor. Return the resulting
-  /// extractor, or `null` if the [node] does not represent a valid value
-  /// extractor.
-  ValueGenerator _translateArgumentExtractor(YamlMap node) {
-    var indexNode = node.valueAt(_indexKey);
-    if (indexNode != null) {
-      _reportUnsupportedKeys(node, const {_indexKey, _kindKey});
-      var index = _translateInteger(
-          indexNode, ErrorContext(key: _indexKey, parentNode: node));
-      if (index == null) {
-        // The error has already been reported.
-        return null;
-      }
-      return CodeFragment(
-          [ArgumentAccessor(PositionalParameterReference(index))]);
-    }
-    var nameNode = node.valueAt(_nameKey);
-    if (nameNode != null) {
-      _reportUnsupportedKeys(node, const {_nameKey, _kindKey});
-      var name = _translateString(
-          nameNode, ErrorContext(key: _nameKey, parentNode: node));
-      if (name == null) {
-        // The error has already been reported.
-        return null;
-      }
-      return CodeFragment([ArgumentAccessor(NamedParameterReference(name))]);
-    }
-    // TODO(brianwilkerson) Report the missing YAML.
-    return null;
-  }
-
   /// Translate the [node] into a bool. Return the resulting bool, or `null`
   /// if the [node] does not represent a valid bool. If the [node] is not
   /// valid, use the [context] to report the error.
@@ -433,6 +412,27 @@
     }
   }
 
+  /// Translate the [node] into a value generator. Return the resulting
+  /// generator, or `null` if the [node] does not represent a valid value
+  /// extractor.
+  ValueGenerator _translateCodeFragment(YamlMap node) {
+    _reportUnsupportedKeys(node, const {_kindKey, _valueKey});
+    var valueNode = node.valueAt(_valueKey);
+    var value = _translateString(
+        valueNode, ErrorContext(key: _valueKey, parentNode: node));
+    if (value == null) {
+      // The error has already been reported.
+      return null;
+    }
+    var accessors = CodeFragmentParser(errorReporter)
+        .parse(value, _offsetOfString(valueNode));
+    if (accessors == null) {
+      // The error has already been reported.
+      return null;
+    }
+    return CodeFragment(accessors);
+  }
+
   /// Translate the [node] into a code template. Return the resulting template,
   /// or `null` if the [node] does not represent a valid code template. If the
   /// [node] is not valid, use the [context] to report the error.
@@ -446,10 +446,7 @@
       if (expressionNode != null) {
         _reportUnsupportedKeys(node, const {_expressionKey, _variablesKey});
         kind = CodeTemplateKind.expression;
-        // TODO(brianwilkerson) We add 1 to account for the quotes around the
-        //  string, but quotes aren't required, so we need to find out what
-        //  style of node [expressionNode] is to get the right offset.
-        templateOffset = expressionNode.span.start.offset + 1;
+        templateOffset = _offsetOfString(expressionNode);
         template = _translateString(expressionNode,
             ErrorContext(key: _expressionKey, parentNode: node));
       } else {
@@ -457,10 +454,7 @@
         if (statementsNode != null) {
           _reportUnsupportedKeys(node, const {_statementsKey, _variablesKey});
           kind = CodeTemplateKind.statements;
-          // TODO(brianwilkerson) We add 1 to account for the quotes around the
-          //  string, but quotes aren't required, so we need to find out what
-          //  style of node [expressionNode] is to get the right offset.
-          templateOffset = statementsNode.span.start.offset + 1;
+          templateOffset = _offsetOfString(statementsNode);
           template = _translateString(statementsNode,
               ErrorContext(key: _statementsKey, parentNode: node));
         } else {
@@ -569,8 +563,8 @@
     }
   }
 
-  /// Translate the [node] into a value extractor. Return the resulting
-  /// extractor, or `null` if the [node] does not represent a valid value
+  /// Translate the [node] into a value generator. Return the resulting
+  /// generator, or `null` if the [node] does not represent a valid value
   /// extractor.
   ValueGenerator _translateImportValue(YamlMap node) {
     _reportUnsupportedKeys(node, const {_kindKey, _nameKey, _urisKey});
@@ -843,13 +837,13 @@
           ErrorContext(key: _kindKey, parentNode: node));
       if (kind == null) {
         return null;
-      } else if (kind == _argumentKind) {
-        return _translateArgumentExtractor(node);
+      } else if (kind == _fragmentKind) {
+        return _translateCodeFragment(node);
       } else if (kind == _importKind) {
         return _translateImportValue(node);
       }
       return _reportInvalidValueOneOf(node, context, [
-        _argumentKind,
+        _fragmentKind,
         _importKind,
       ]);
     } else if (node == null) {
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/code_fragment_parser_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/code_fragment_parser_test.dart
new file mode 100644
index 0000000..719eb33
--- /dev/null
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/code_fragment_parser_test.dart
@@ -0,0 +1,77 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:_fe_analyzer_shared/src/base/errors.dart';
+import 'package:analysis_server/src/services/correction/fix/data_driven/accessor.dart';
+import 'package:analysis_server/src/services/correction/fix/data_driven/code_fragment_parser.dart';
+import 'package:analyzer/error/listener.dart';
+import 'package:matcher/matcher.dart';
+import 'package:test/test.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+import '../../../../../mocks.dart';
+import '../../../../../utils/test_support.dart';
+
+void main() {
+  defineReflectiveSuite(() {
+    defineReflectiveTests(CodeFragmentParserTest);
+  });
+}
+
+abstract class AbstractCodeFragmentParserTest {
+  List<Accessor> assertErrors(
+      String content, List<ExpectedError> expectedErrors) {
+    var errorListener = GatheringErrorListener();
+    var errorReporter = ErrorReporter(errorListener, MockSource());
+    var accessors = CodeFragmentParser(errorReporter).parse(content, 0);
+    errorListener.assertErrors(expectedErrors);
+    return accessors;
+  }
+
+  List<Accessor> assertNoErrors(String content) {
+    var errorListener = GatheringErrorListener();
+    var errorReporter = ErrorReporter(errorListener, MockSource());
+    var accessors = CodeFragmentParser(errorReporter).parse(content, 0);
+    errorListener.assertNoErrors();
+    return accessors;
+  }
+
+  ExpectedError error(ErrorCode code, int offset, int length,
+          {String message,
+          Pattern messageContains,
+          List<ExpectedContextMessage> contextMessages =
+              const <ExpectedContextMessage>[]}) =>
+      ExpectedError(code, offset, length,
+          message: message,
+          messageContains: messageContains,
+          expectedContextMessages: contextMessages);
+}
+
+@reflectiveTest
+class CodeFragmentParserTest extends AbstractCodeFragmentParserTest {
+  void test_arguments_arguments_arguments() {
+    var accessors = assertNoErrors('arguments[0].arguments[1].arguments[2]');
+    expect(accessors, hasLength(3));
+  }
+
+  void test_arguments_named() {
+    var accessors = assertNoErrors('arguments[foo]');
+    expect(accessors, hasLength(1));
+  }
+
+  void test_arguments_positional() {
+    var accessors = assertNoErrors('arguments[0]');
+    expect(accessors, hasLength(1));
+  }
+
+  void test_arguments_typeArguments() {
+    var accessors = assertNoErrors('arguments[0].typeArguments[0]');
+    expect(accessors, hasLength(2));
+  }
+
+  void test_typeArguments() {
+    var accessors = assertNoErrors('typeArguments[0]');
+    expect(accessors, hasLength(1));
+  }
+}
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/invalid_character_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/invalid_character_test.dart
new file mode 100644
index 0000000..ab3ece6
--- /dev/null
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/invalid_character_test.dart
@@ -0,0 +1,29 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/src/services/correction/fix/data_driven/transform_set_error_code.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+import '../code_fragment_parser_test.dart';
+
+void main() {
+  defineReflectiveSuite(() {
+    defineReflectiveTests(InvalidCharacterTest);
+  });
+}
+
+@reflectiveTest
+class InvalidCharacterTest extends AbstractCodeFragmentParserTest {
+  void test_final() {
+    assertErrors('arguments;', [
+      error(TransformSetErrorCode.invalidCharacter, 9, 1),
+    ]);
+  }
+
+  void test_initial() {
+    assertErrors('{ some', [
+      error(TransformSetErrorCode.invalidCharacter, 0, 1),
+    ]);
+  }
+}
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/missing_token_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/missing_token_test.dart
new file mode 100644
index 0000000..ec8ac42
--- /dev/null
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/missing_token_test.dart
@@ -0,0 +1,47 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/src/services/correction/fix/data_driven/transform_set_error_code.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+import '../code_fragment_parser_test.dart';
+
+void main() {
+  defineReflectiveSuite(() {
+    defineReflectiveTests(MissingTokenTest);
+  });
+}
+
+@reflectiveTest
+class MissingTokenTest extends AbstractCodeFragmentParserTest {
+  void test_closeBracket() {
+    assertErrors('arguments[2', [
+      error(TransformSetErrorCode.missingToken, 10, 1),
+    ]);
+  }
+
+  void test_identifier_afterPeriod() {
+    assertErrors('arguments[2].', [
+      error(TransformSetErrorCode.missingToken, 12, 1),
+    ]);
+  }
+
+  void test_identifier_initial() {
+    assertErrors('', [
+      error(TransformSetErrorCode.missingToken, 0, 0),
+    ]);
+  }
+
+  void test_index() {
+    assertErrors('arguments[', [
+      error(TransformSetErrorCode.missingToken, 9, 1),
+    ]);
+  }
+
+  void test_openBracket() {
+    assertErrors('arguments', [
+      error(TransformSetErrorCode.missingToken, 0, 9),
+    ]);
+  }
+}
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/test_all.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/test_all.dart
index e8bbf94..d8b87c6 100644
--- a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/test_all.dart
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/test_all.dart
@@ -4,22 +4,30 @@
 
 import 'package:test_reflective_loader/test_reflective_loader.dart';
 
+import 'invalid_character_test.dart' as invalid_character;
 import 'invalid_value_one_of_test.dart' as invalid_value_one_of;
 import 'invalid_value_test.dart' as invalid_value;
 import 'missing_key_test.dart' as missing_key;
 import 'missing_template_end_test.dart' as missing_template_end;
+import 'missing_token_test.dart' as missing_token;
 import 'undefined_variable_test.dart' as undefined_variable;
+import 'unknown_accessor_test.dart' as unknown_accessor;
 import 'unsupported_key_test.dart' as unsupported_key;
+import 'wrong_token_test.dart' as wrong_token;
 import 'yaml_syntax_error_test.dart' as yaml_syntax_error;
 
 void main() {
   defineReflectiveSuite(() {
+    invalid_character.main();
     invalid_value_one_of.main();
     invalid_value.main();
     missing_key.main();
     missing_template_end.main();
+    missing_token.main();
     undefined_variable.main();
+    unknown_accessor.main();
     unsupported_key.main();
+    wrong_token.main();
     yaml_syntax_error.main();
   });
 }
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/undefined_variable_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/undefined_variable_test.dart
index 19334c8..db854a5 100644
--- a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/undefined_variable_test.dart
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/undefined_variable_test.dart
@@ -33,8 +33,8 @@
         expression: '{%xyz%}'
         variables:
           zyx:
-            kind: 'argument'
-            index: 0
+            kind: 'fragment'
+            value: 'arguments[0]'
 ''', [
       error(TransformSetErrorCode.undefinedVariable, 253, 3),
     ]);
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/unknown_accessor_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/unknown_accessor_test.dart
new file mode 100644
index 0000000..205ad42
--- /dev/null
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/unknown_accessor_test.dart
@@ -0,0 +1,29 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/src/services/correction/fix/data_driven/transform_set_error_code.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+import '../code_fragment_parser_test.dart';
+
+void main() {
+  defineReflectiveSuite(() {
+    defineReflectiveTests(UnknownAccessorTest);
+  });
+}
+
+@reflectiveTest
+class UnknownAccessorTest extends AbstractCodeFragmentParserTest {
+  void test_afterPeriod() {
+    assertErrors('arguments[0].argument[1]', [
+      error(TransformSetErrorCode.unknownAccessor, 13, 8),
+    ]);
+  }
+
+  void test_initial() {
+    assertErrors('argument[0]', [
+      error(TransformSetErrorCode.unknownAccessor, 0, 8),
+    ]);
+  }
+}
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/wrong_token_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/wrong_token_test.dart
new file mode 100644
index 0000000..a047759
--- /dev/null
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/diagnostics/wrong_token_test.dart
@@ -0,0 +1,53 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/src/services/correction/fix/data_driven/transform_set_error_code.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+import '../code_fragment_parser_test.dart';
+
+void main() {
+  defineReflectiveSuite(() {
+    defineReflectiveTests(MissingTokenTest);
+  });
+}
+
+@reflectiveTest
+class MissingTokenTest extends AbstractCodeFragmentParserTest {
+  void test_closeBracket() {
+    assertErrors('arguments[2 3', [
+      error(TransformSetErrorCode.wrongToken, 12, 1),
+    ]);
+  }
+
+  void test_identifier_afterPeriod() {
+    assertErrors('arguments[2].1', [
+      error(TransformSetErrorCode.wrongToken, 13, 1),
+    ]);
+  }
+
+  void test_identifier_initial() {
+    assertErrors('1', [
+      error(TransformSetErrorCode.wrongToken, 0, 1),
+    ]);
+  }
+
+  void test_index() {
+    assertErrors('arguments[.', [
+      error(TransformSetErrorCode.wrongToken, 10, 1),
+    ]);
+  }
+
+  void test_openBracket() {
+    assertErrors('arguments.', [
+      error(TransformSetErrorCode.wrongToken, 9, 1),
+    ]);
+  }
+
+  void test_period() {
+    assertErrors('arguments[2] typeArguments', [
+      error(TransformSetErrorCode.wrongToken, 13, 13),
+    ]);
+  }
+}
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/end_to_end_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/end_to_end_test.dart
index aad57fa..df98271 100644
--- a/pkg/analysis_server/test/src/services/correction/fix/data_driven/end_to_end_test.dart
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/end_to_end_test.dart
@@ -38,8 +38,8 @@
         expression: '{% y %}'
         variables:
           y:
-            kind: 'argument'
-            index: 0
+            kind: 'fragment'
+            value: 'arguments[0]'
 ''');
     await resolveTestUnit('''
 import '$importUri';
@@ -81,8 +81,8 @@
         expression: '{% t %}'
         variables:
           t:
-            kind: 'argument'
-            index: 0
+            kind: 'fragment'
+            value: 'arguments[0]'
 ''');
     await resolveTestUnit('''
 import '$importUri';
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/flutter_use_case_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/flutter_use_case_test.dart
index c26ea18..074330b 100644
--- a/pkg/analysis_server/test/src/services/correction/fix/data_driven/flutter_use_case_test.dart
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/flutter_use_case_test.dart
@@ -847,8 +847,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -894,8 +894,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -943,8 +943,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -990,8 +990,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1111,8 +1111,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1158,8 +1158,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1207,8 +1207,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1254,8 +1254,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1303,8 +1303,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1349,8 +1349,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1470,8 +1470,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
@@ -1517,8 +1517,8 @@
           expression: '{% type %}'
           variables:
             type:
-              kind: 'argument'
-              index: 0
+              kind: 'fragment'
+              value: 'arguments[0]'
       - kind: 'removeParameter'
         index: 0
 ''');
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/test_all.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/test_all.dart
index 5f2419e..28ee44c 100644
--- a/pkg/analysis_server/test/src/services/correction/fix/data_driven/test_all.dart
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/test_all.dart
@@ -5,6 +5,7 @@
 import 'package:test_reflective_loader/test_reflective_loader.dart';
 
 import 'add_type_parameter_test.dart' as add_type_parameter_change;
+import 'code_fragment_parser_test.dart' as code_fragment_parser;
 import 'code_template_test.dart' as code_template;
 import 'diagnostics/test_all.dart' as diagnostics;
 import 'end_to_end_test.dart' as end_to_end;
@@ -17,6 +18,7 @@
 void main() {
   defineReflectiveSuite(() {
     add_type_parameter_change.main();
+    code_fragment_parser.main();
     code_template.main();
     diagnostics.main();
     end_to_end.main();
diff --git a/pkg/analysis_server/test/src/services/correction/fix/data_driven/transform_set_parser_test.dart b/pkg/analysis_server/test/src/services/correction/fix/data_driven/transform_set_parser_test.dart
index 0e526cb..e28220e 100644
--- a/pkg/analysis_server/test/src/services/correction/fix/data_driven/transform_set_parser_test.dart
+++ b/pkg/analysis_server/test/src/services/correction/fix/data_driven/transform_set_parser_test.dart
@@ -46,8 +46,8 @@
         expression: '{% p %}'
         variables:
           p:
-            kind: 'argument'
-            index: 1
+            kind: 'fragment'
+            value: 'arguments[1]'
 ''');
     var transforms = _transforms('f');
     expect(transforms, hasLength(1));
@@ -117,8 +117,8 @@
         expression: '{% p %}'
         variables:
           p:
-            kind: 'argument'
-            index: 1
+            kind: 'fragment'
+            value: 'arguments[1]'
 ''');
     var transforms = _transforms('f');
     expect(transforms, hasLength(1));
@@ -158,8 +158,8 @@
         expression: '{% p %}'
         variables:
           p:
-            kind: 'argument'
-            index: 1
+            kind: 'fragment'
+            value: 'arguments[1]'
 ''');
     var transforms = _transforms('f');
     expect(transforms, hasLength(1));
@@ -199,11 +199,11 @@
         expression: '{% a %}({% b %})'
         variables:
           a:
-            kind: 'argument'
-            index: 1
+            kind: 'fragment'
+            value: 'arguments[1]'
           b:
-            kind: 'argument'
-            index: 2
+            kind: 'fragment'
+            value: 'arguments[2]'
 ''');
     var transforms = _transforms('f');
     expect(transforms, hasLength(1));
@@ -285,8 +285,8 @@
         expression: '{% t %}'
         variables:
           t:
-            kind: 'argument'
-            name: 'p'
+            kind: 'fragment'
+            value: 'arguments[p]'
 ''');
     var transforms = _transforms('A');
     expect(transforms, hasLength(1));
@@ -325,8 +325,8 @@
         expression: '{% t %}'
         variables:
           t:
-            kind: 'argument'
-            index: 2
+            kind: 'fragment'
+            value: 'arguments[2]'
 ''');
     var transforms = _transforms('A');
     expect(transforms, hasLength(1));