utils/template/tokenizer.dart

// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

class Tokenizer extends TokenizerBase {
  TokenKind tmplTokens;

  bool _selectorParsing;

  Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0])
    : super(source, skipWhitespace, index), _selectorParsing = false {
      tmplTokens = new TokenKind();
  }

  int get startIndex => _startIndex;
  void set index(int idx) {
    _index = idx;
  }

  Token next([bool inTag = true]) {
    // keep track of our starting position
    _startIndex = _index;

    if (_interpStack != null && _interpStack.depth == 0) {
      var istack = _interpStack;
      _interpStack = _interpStack.pop();

      /* TODO(terry): Enable for variable and string interpolation.
       * if (istack.isMultiline) {
       *   return finishMultilineStringBody(istack.quote);
       * } else {
       *   return finishStringBody(istack.quote);
       * }
       */
    }

    int ch;
    ch = _nextChar();
    switch(ch) {
      case 0:
        return _finishToken(TokenKind.END_OF_FILE);
      case tmplTokens.tokens[TokenKind.SPACE]:
      case tmplTokens.tokens[TokenKind.TAB]:
      case tmplTokens.tokens[TokenKind.NEWLINE]:
      case tmplTokens.tokens[TokenKind.RETURN]:
        if (inTag) {
          return finishWhitespace();
        } else {
          return _finishToken(TokenKind.WHITESPACE);
        }
      case tmplTokens.tokens[TokenKind.END_OF_FILE]:
        return _finishToken(TokenKind.END_OF_FILE);
      case tmplTokens.tokens[TokenKind.LPAREN]:
        return _finishToken(TokenKind.LPAREN);
      case tmplTokens.tokens[TokenKind.RPAREN]:
        return _finishToken(TokenKind.RPAREN);
      case tmplTokens.tokens[TokenKind.COMMA]:
        return _finishToken(TokenKind.COMMA);
      case tmplTokens.tokens[TokenKind.LBRACE]:
        return _finishToken(TokenKind.LBRACE);
      case tmplTokens.tokens[TokenKind.RBRACE]:
        return _finishToken(TokenKind.RBRACE);
      case tmplTokens.tokens[TokenKind.LESS_THAN]:
        return _finishToken(TokenKind.LESS_THAN);
      case tmplTokens.tokens[TokenKind.GREATER_THAN]:
        return _finishToken(TokenKind.GREATER_THAN);
      case tmplTokens.tokens[TokenKind.EQUAL]:
        if (inTag) {
          if (_maybeEatChar(tmplTokens.tokens[TokenKind.SINGLE_QUOTE])) {
            return finishQuotedAttrValue(
              tmplTokens.tokens[TokenKind.SINGLE_QUOTE]);
          } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.DOUBLE_QUOTE])) {
            return finishQuotedAttrValue(
              tmplTokens.tokens[TokenKind.DOUBLE_QUOTE]);
          } else if (TokenizerHelpers.isAttributeValueStart(_peekChar())) {
            return finishAttrValue();
          }
        }
        return _finishToken(TokenKind.EQUAL);
      case tmplTokens.tokens[TokenKind.SLASH]:
        if (_maybeEatChar(tmplTokens.tokens[TokenKind.GREATER_THAN])) {
          return _finishToken(TokenKind.END_NO_SCOPE_TAG);          // />
        } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.ASTERISK])) {
          return finishMultiLineComment();
        } else {
          return _finishToken(TokenKind.SLASH);
        }
      case tmplTokens.tokens[TokenKind.DOLLAR]:
        if (_maybeEatChar(tmplTokens.tokens[TokenKind.LBRACE])) {
          if (_maybeEatChar(tmplTokens.tokens[TokenKind.HASH])) {
            return _finishToken(TokenKind.START_COMMAND);           // ${#
          } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.SLASH])) {
            return _finishToken(TokenKind.END_COMMAND);             // ${/
          } else {
            return _finishToken(TokenKind.START_EXPRESSION);        // ${
          }
        } else {
          return _finishToken(TokenKind.DOLLAR);
        }

      default:
        if (TokenizerHelpers.isIdentifierStart(ch)) {
          return this.finishIdentifier();
        } else if (TokenizerHelpers.isDigit(ch)) {
          return this.finishNumber();
        } else {
          return _errorToken();
        }
    }
  }

  // TODO(jmesserly): we need a way to emit human readable error messages from
  // the tokenizer.
  Token _errorToken([String message = null]) {
    return _finishToken(TokenKind.ERROR);
  }

  int getIdentifierKind() {
    // Is the identifier an element?
    int tokId = TokenKind.matchElements(_text, _startIndex,
      _index - _startIndex);
    if (tokId == -1) {
      // No, is it an attribute?
//      tokId = TokenKind.matchAttributes(_text, _startIndex, _index - _startIndex);
    }
    if (tokId == -1) {
      tokId = TokenKind.matchKeywords(_text, _startIndex, _index - _startIndex);
    }

    return tokId >= 0 ? tokId : TokenKind.IDENTIFIER;
  }

  // Need to override so CSS version of isIdentifierPart is used.
  Token finishIdentifier() {
    while (_index < _text.length) {
//      if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) {
      if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index))) {
//        _index--;
        break;
      } else {
        _index += 1;
      }
    }
    if (_interpStack != null && _interpStack.depth == -1) {
      _interpStack.depth = 0;
    }
    int kind = getIdentifierKind();
    if (kind == TokenKind.IDENTIFIER) {
      return _finishToken(TokenKind.IDENTIFIER);
    } else {
      return _finishToken(kind);
    }
  }

  Token _makeAttributeValueToken(List<int> buf) {
    final s = new String.fromCharCodes(buf);
    return new LiteralToken(TokenKind.ATTR_VALUE, _source, _startIndex, _index,
      s);
  }

  /* quote if -1 signals to read upto first whitespace otherwise read upto
   * single or double quote char.
   */
  Token finishQuotedAttrValue([int quote = -1]) {
    var buf = new List<int>();
    while (true) {
      int ch = _nextChar();
      if (ch == quote) {
        return _makeAttributeValueToken(buf);
      } else if (ch == 0) {
        return _errorToken();
      } else {
        buf.add(ch);
      }
    }
  }

  Token finishAttrValue() {
    var buf = new List<int>();
    while (true) {
      int ch = _peekChar();
      if (TokenizerHelpers.isWhitespace(ch) || TokenizerHelpers.isSlash(ch) ||
          TokenizerHelpers.isCloseTag(ch)) {
        return _makeAttributeValueToken(buf);
      } else if (ch == 0) {
        return _errorToken();
      } else {
        buf.add(_nextChar());
      }
    }
  }

  Token finishNumber() {
    eatDigits();

    if (_peekChar() == 46/*.*/) {
      // Handle the case of 1.toString().
      _nextChar();
      if (TokenizerHelpers.isDigit(_peekChar())) {
        eatDigits();
        return _finishToken(TokenKind.DOUBLE);
      } else {
        _index -= 1;
      }
    }

    return _finishToken(TokenKind.INTEGER);
  }

  bool maybeEatDigit() {
    if (_index < _text.length && TokenizerHelpers.isDigit(
        _text.codeUnitAt(_index))) {
      _index += 1;
      return true;
    }
    return false;
  }

  void eatHexDigits() {
    while (_index < _text.length) {
     if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) {
       _index += 1;
     } else {
       return;
     }
    }
  }

  bool maybeEatHexDigit() {
    if (_index < _text.length && TokenizerHelpers.isHexDigit(
        _text.codeUnitAt(_index))) {
      _index += 1;
      return true;
    }
    return false;
  }

  Token finishMultiLineComment() {
    while (true) {
      int ch = _nextChar();
      if (ch == 0) {
        return _finishToken(TokenKind.INCOMPLETE_COMMENT);
      } else if (ch == 42/*'*'*/) {
        if (_maybeEatChar(47/*'/'*/)) {
          if (_skipWhitespace) {
            return next();
          } else {
            return _finishToken(TokenKind.COMMENT);
          }
        }
      } else if (ch == tmplTokens.tokens[TokenKind.MINUS]) {
        /* Check if close part of Comment Definition --> (CDC). */
        if (_maybeEatChar(tmplTokens.tokens[TokenKind.MINUS])) {
          if (_maybeEatChar(tmplTokens.tokens[TokenKind.GREATER_THAN])) {
            if (_skipWhitespace) {
              return next();
            } else {
              return _finishToken(TokenKind.HTML_COMMENT);
            }
          }
        }
      }
    }
    return _errorToken();
  }

}


/** Static helper methods. */
class TokenizerHelpers {
  static bool isIdentifierStart(int c) {
    return ((c >= 97/*a*/ && c <= 122/*z*/) ||
        (c >= 65/*A*/ && c <= 90/*Z*/) || c == 95/*_*/);
  }

  static bool isDigit(int c) {
    return (c >= 48/*0*/ && c <= 57/*9*/);
  }

  static bool isHexDigit(int c) {
    return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) ||
        (c >= 65/*A*/ && c <= 70/*F*/));
  }

  static bool isWhitespace(int c) {
    return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ ||
        c == 13/*'\r'*/);
  }

  static bool isIdentifierPart(int c) {
    return (isIdentifierStart(c) || isDigit(c) || c == 45/*-*/ ||
        c == 58/*:*/ || c == 46/*.*/);
  }

  static bool isInterpIdentifierPart(int c) {
    return (isIdentifierStart(c) || isDigit(c));
  }

  static bool isAttributeValueStart(int c) {
    return !isWhitespace(c) && !isSlash(c) && !isCloseTag(c);
  }

  static bool isSlash(int c) {
    return (c == 47/* / */);
  }

  static bool isCloseTag(int c) {
    return (c == 62/* > */);
  }
}