| // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| class Tokenizer extends CSSTokenizerBase { |
| TokenKind cssTokens; |
| |
| bool _selectorParsing; |
| |
| Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) |
| : super(source, skipWhitespace, index), _selectorParsing = false { |
| cssTokens = new TokenKind(); |
| } |
| |
| int get startIndex => _startIndex; |
| |
| Token next() { |
| // keep track of our starting position |
| _startIndex = _index; |
| |
| if (_interpStack != null && _interpStack.depth == 0) { |
| var istack = _interpStack; |
| _interpStack = _interpStack.pop(); |
| |
| /* TODO(terry): Enable for variable and string interpolation. |
| * if (istack.isMultiline) { |
| * return finishMultilineStringBody(istack.quote); |
| * } else { |
| * return finishStringBody(istack.quote); |
| * } |
| */ |
| } |
| |
| int ch; |
| ch = _nextChar(); |
| switch(ch) { |
| case 0: |
| return _finishToken(TokenKind.END_OF_FILE); |
| case cssTokens.tokens[TokenKind.SPACE]: |
| case cssTokens.tokens[TokenKind.TAB]: |
| case cssTokens.tokens[TokenKind.NEWLINE]: |
| case cssTokens.tokens[TokenKind.RETURN]: |
| return finishWhitespace(); |
| case cssTokens.tokens[TokenKind.END_OF_FILE]: |
| return _finishToken(TokenKind.END_OF_FILE); |
| case cssTokens.tokens[TokenKind.AT]: |
| return _finishToken(TokenKind.AT); |
| case cssTokens.tokens[TokenKind.DOT]: |
| int start = _startIndex; // Start where the dot started. |
| if (maybeEatDigit()) { |
| // looks like a number dot followed by digit(s). |
| Token number = finishNumber(); |
| if (number.kind == TokenKind.INTEGER) { |
| // It's a number but it's preceeded by a dot, so make it a double. |
| _startIndex = start; |
| return _finishToken(TokenKind.DOUBLE); |
| } else { |
| // Don't allow dot followed by a double (e.g, '..1'). |
| return _errorToken(); |
| } |
| } else { |
| // It's really a dot. |
| return _finishToken(TokenKind.DOT); |
| } |
| case cssTokens.tokens[TokenKind.LPAREN]: |
| return _finishToken(TokenKind.LPAREN); |
| case cssTokens.tokens[TokenKind.RPAREN]: |
| return _finishToken(TokenKind.RPAREN); |
| case cssTokens.tokens[TokenKind.LBRACE]: |
| return _finishToken(TokenKind.LBRACE); |
| case cssTokens.tokens[TokenKind.RBRACE]: |
| return _finishToken(TokenKind.RBRACE); |
| case cssTokens.tokens[TokenKind.LBRACK]: |
| return _finishToken(TokenKind.LBRACK); |
| case cssTokens.tokens[TokenKind.RBRACK]: |
| return _finishToken(TokenKind.RBRACK); |
| case cssTokens.tokens[TokenKind.HASH]: |
| return _finishToken(TokenKind.HASH); |
| case cssTokens.tokens[TokenKind.PLUS]: |
| if (maybeEatDigit()) { |
| return finishNumber(); |
| } else { |
| return _finishToken(TokenKind.PLUS); |
| } |
| case cssTokens.tokens[TokenKind.MINUS]: |
| if (maybeEatDigit()) { |
| return finishNumber(); |
| } else if (TokenizerHelpers.isIdentifierStart(ch)) { |
| return this.finishIdentifier(ch); |
| } else { |
| return _finishToken(TokenKind.MINUS); |
| } |
| case cssTokens.tokens[TokenKind.GREATER]: |
| return _finishToken(TokenKind.GREATER); |
| case cssTokens.tokens[TokenKind.TILDE]: |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { |
| return _finishToken(TokenKind.INCLUDES); // ~= |
| } else { |
| return _finishToken(TokenKind.TILDE); |
| } |
| case cssTokens.tokens[TokenKind.ASTERISK]: |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { |
| return _finishToken(TokenKind.SUBSTRING_MATCH); // *= |
| } else { |
| return _finishToken(TokenKind.ASTERISK); |
| } |
| case cssTokens.tokens[TokenKind.NAMESPACE]: |
| return _finishToken(TokenKind.NAMESPACE); |
| case cssTokens.tokens[TokenKind.COLON]: |
| return _finishToken(TokenKind.COLON); |
| case cssTokens.tokens[TokenKind.COMMA]: |
| return _finishToken(TokenKind.COMMA); |
| case cssTokens.tokens[TokenKind.SEMICOLON]: |
| return _finishToken(TokenKind.SEMICOLON); |
| case cssTokens.tokens[TokenKind.PERCENT]: |
| return _finishToken(TokenKind.PERCENT); |
| case cssTokens.tokens[TokenKind.SINGLE_QUOTE]: |
| return _finishToken(TokenKind.SINGLE_QUOTE); |
| case cssTokens.tokens[TokenKind.DOUBLE_QUOTE]: |
| return _finishToken(TokenKind.DOUBLE_QUOTE); |
| case cssTokens.tokens[TokenKind.SLASH]: |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.ASTERISK])) { |
| return finishMultiLineComment(); |
| } else { |
| return _finishToken(TokenKind.SLASH); |
| } |
| case cssTokens.tokens[TokenKind.LESS]: // <!-- |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.BANG]) && |
| _maybeEatChar(cssTokens.tokens[TokenKind.MINUS]) && |
| _maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) { |
| return finishMultiLineComment(); |
| } else { |
| return _finishToken(TokenKind.LESS); |
| } |
| case cssTokens.tokens[TokenKind.EQUALS]: |
| return _finishToken(TokenKind.EQUALS); |
| case cssTokens.tokens[TokenKind.OR]: |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { |
| return _finishToken(TokenKind.DASH_MATCH); // |= |
| } else { |
| return _finishToken(TokenKind.OR); |
| } |
| case cssTokens.tokens[TokenKind.CARET]: |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { |
| return _finishToken(TokenKind.PREFIX_MATCH); // ^= |
| } else { |
| return _finishToken(TokenKind.CARET); |
| } |
| case cssTokens.tokens[TokenKind.DOLLAR]: |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { |
| return _finishToken(TokenKind.SUFFIX_MATCH); // $= |
| } else { |
| return _finishToken(TokenKind.DOLLAR); |
| } |
| case cssTokens.tokens[TokenKind.BANG]: |
| Token tok = finishIdentifier(ch); |
| return (tok == null) ? _finishToken(TokenKind.BANG) : tok; |
| default: |
| if (TokenizerHelpers.isIdentifierStart(ch)) { |
| return this.finishIdentifier(ch); |
| } else if (TokenizerHelpers.isDigit(ch)) { |
| return this.finishNumber(); |
| } else { |
| return _errorToken(); |
| } |
| } |
| } |
| |
| // TODO(jmesserly): we need a way to emit human readable error messages from |
| // the tokenizer. |
| Token _errorToken([String message = null]) { |
| return _finishToken(TokenKind.ERROR); |
| } |
| |
| int getIdentifierKind() { |
| // Is the identifier a unit type? |
| int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex); |
| if (tokId == -1) { |
| // No, is it a directive? |
| tokId = TokenKind.matchDirectives( |
| _text, _startIndex, _index - _startIndex); |
| } |
| if (tokId == -1) { |
| tokId = (_text.substring(_startIndex, _index) == '!important') ? |
| TokenKind.IMPORTANT : -1; |
| } |
| |
| return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; |
| } |
| |
| // Need to override so CSS version of isIdentifierPart is used. |
| Token finishIdentifier(int ch) { |
| while (_index < _text.length) { |
| // if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) { |
| if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index))) { |
| // _index--; |
| break; |
| } else { |
| _index += 1; |
| } |
| } |
| if (_interpStack != null && _interpStack.depth == -1) { |
| _interpStack.depth = 0; |
| } |
| int kind = getIdentifierKind(); |
| if (kind == TokenKind.IDENTIFIER) { |
| return _finishToken(TokenKind.IDENTIFIER); |
| } else { |
| return _finishToken(kind); |
| } |
| } |
| |
| Token finishImportant() { |
| |
| } |
| |
| Token finishNumber() { |
| eatDigits(); |
| |
| if (_peekChar() == 46/*.*/) { |
| // Handle the case of 1.toString(). |
| _nextChar(); |
| if (TokenizerHelpers.isDigit(_peekChar())) { |
| eatDigits(); |
| return _finishToken(TokenKind.DOUBLE); |
| } else { |
| _index -= 1; |
| } |
| } |
| |
| return _finishToken(TokenKind.INTEGER); |
| } |
| |
| bool maybeEatDigit() { |
| if (_index < _text.length |
| && TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { |
| _index += 1; |
| return true; |
| } |
| return false; |
| } |
| |
| void eatHexDigits() { |
| while (_index < _text.length) { |
| if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { |
| _index += 1; |
| } else { |
| return; |
| } |
| } |
| } |
| |
| bool maybeEatHexDigit() { |
| if (_index < _text.length |
| && TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { |
| _index += 1; |
| return true; |
| } |
| return false; |
| } |
| |
| Token finishMultiLineComment() { |
| while (true) { |
| int ch = _nextChar(); |
| if (ch == 0) { |
| return _finishToken(TokenKind.INCOMPLETE_COMMENT); |
| } else if (ch == 42/*'*'*/) { |
| if (_maybeEatChar(47/*'/'*/)) { |
| if (_skipWhitespace) { |
| return next(); |
| } else { |
| return _finishToken(TokenKind.COMMENT); |
| } |
| } |
| } else if (ch == cssTokens.tokens[TokenKind.MINUS]) { |
| /* Check if close part of Comment Definition --> (CDC). */ |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) { |
| if (_maybeEatChar(cssTokens.tokens[TokenKind.GREATER])) { |
| if (_skipWhitespace) { |
| return next(); |
| } else { |
| return _finishToken(TokenKind.HTML_COMMENT); |
| } |
| } |
| } |
| } |
| } |
| return _errorToken(); |
| } |
| |
| } |
| |
| /** Static helper methods. */ |
| /** Static helper methods. */ |
| class TokenizerHelpers { |
| |
| static bool isIdentifierStart(int c) { |
| return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || |
| c == 95/*_*/ || c == 45 /*-*/); |
| } |
| |
| static bool isDigit(int c) { |
| return (c >= 48/*0*/ && c <= 57/*9*/); |
| } |
| |
| static bool isHexDigit(int c) { |
| return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) |
| || (c >= 65/*A*/ && c <= 70/*F*/)); |
| } |
| |
| static bool isIdentifierPart(int c) { |
| return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/); |
| } |
| } |
| |
