| library tokenizer_test; |
| |
| // Note: mirrors used to match the getattr usage in the original test |
| import 'dart:convert'; |
| import 'dart:io'; |
| import 'dart:mirrors'; |
| import 'package:path/path.dart' as pathos; |
| import 'package:unittest/unittest.dart'; |
| import 'package:html/src/char_encodings.dart'; |
| import 'package:html/src/token.dart'; |
| import 'package:html/src/tokenizer.dart'; |
| import 'package:utf/utf.dart'; |
| import 'support.dart'; |
| |
| class TokenizerTestParser { |
| String _state; |
| var _lastStartTag; |
| List outputTokens; |
| |
| TokenizerTestParser(String initialState, [lastStartTag]) |
| : _state = initialState, |
| _lastStartTag = lastStartTag; |
| |
| List parse(String str) { |
| // Note: we need to pass bytes to the tokenizer if we want it to handle BOM. |
| var bytes = codepointsToUtf8(toCodepoints(str)); |
| var tokenizer = new HtmlTokenizer(bytes, encoding: 'utf-8'); |
| outputTokens = []; |
| |
| // Note: we can't get a closure of the state method. However, we can |
| // create a new closure to invoke it via mirrors. |
| var mtok = reflect(tokenizer); |
| tokenizer.state = () => mtok.invoke(new Symbol(_state), const []).reflectee; |
| |
| if (_lastStartTag != null) { |
| tokenizer.currentToken = new StartTagToken(_lastStartTag); |
| } |
| |
| while (tokenizer.moveNext()) { |
| var token = tokenizer.current; |
| switch (token.kind) { |
| case TokenKind.characters: |
| processCharacters(token); |
| break; |
| case TokenKind.spaceCharacters: |
| processSpaceCharacters(token); |
| break; |
| case TokenKind.startTag: |
| processStartTag(token); |
| break; |
| case TokenKind.endTag: |
| processEndTag(token); |
| break; |
| case TokenKind.comment: |
| processComment(token); |
| break; |
| case TokenKind.doctype: |
| processDoctype(token); |
| break; |
| case TokenKind.parseError: |
| processParseError(token); |
| break; |
| } |
| } |
| |
| return outputTokens; |
| } |
| |
| void processDoctype(DoctypeToken token) { |
| outputTokens.add( |
| ["DOCTYPE", token.name, token.publicId, token.systemId, token.correct]); |
| } |
| |
| void processStartTag(StartTagToken token) { |
| outputTokens.add(["StartTag", token.name, token.data, token.selfClosing]); |
| } |
| |
| void processEndTag(EndTagToken token) { |
| outputTokens.add(["EndTag", token.name, token.selfClosing]); |
| } |
| |
| void processComment(StringToken token) { |
| outputTokens.add(["Comment", token.data]); |
| } |
| |
| void processSpaceCharacters(StringToken token) { |
| processCharacters(token); |
| } |
| |
| void processCharacters(StringToken token) { |
| outputTokens.add(["Character", token.data]); |
| } |
| |
| void processEOF(token) {} |
| |
| void processParseError(StringToken token) { |
| // TODO(jmesserly): when debugging test failures it can be useful to add |
| // logging here like `print('ParseError $token');`. It would be nice to |
| // use the actual logging library. |
| outputTokens.add(["ParseError", token.data]); |
| } |
| } |
| |
| List concatenateCharacterTokens(List tokens) { |
| var outputTokens = []; |
| for (var token in tokens) { |
| if (token.indexOf("ParseError") == -1 && token[0] == "Character") { |
| if (outputTokens.length > 0 && |
| outputTokens.last.indexOf("ParseError") == -1 && |
| outputTokens.last[0] == "Character") { |
| outputTokens.last[1] = '${outputTokens.last[1]}${token[1]}'; |
| } else { |
| outputTokens.add(token); |
| } |
| } else { |
| outputTokens.add(token); |
| } |
| } |
| return outputTokens; |
| } |
| |
| List normalizeTokens(List tokens) { |
| // TODO: convert tests to reflect arrays |
| for (int i = 0; i < tokens.length; i++) { |
| var token = tokens[i]; |
| if (token[0] == 'ParseError') { |
| tokens[i] = token[0]; |
| } |
| } |
| return tokens; |
| } |
| |
| /// Test whether the test has passed or failed |
| /// |
| /// If the ignoreErrorOrder flag is set to true we don't test the relative |
| /// positions of parse errors and non parse errors. |
| void expectTokensMatch( |
| List expectedTokens, List receivedTokens, bool ignoreErrorOrder, |
| [bool ignoreErrors = false, String message]) { |
| var checkSelfClosing = false; |
| for (var token in expectedTokens) { |
| if (token[0] == "StartTag" && token.length == 4 || |
| token[0] == "EndTag" && token.length == 3) { |
| checkSelfClosing = true; |
| break; |
| } |
| } |
| |
| if (!checkSelfClosing) { |
| for (var token in receivedTokens) { |
| if (token[0] == "StartTag" || token[0] == "EndTag") { |
| token.removeLast(); |
| } |
| } |
| } |
| |
| if (!ignoreErrorOrder && !ignoreErrors) { |
| expect(receivedTokens, equals(expectedTokens), reason: message); |
| } else { |
| // Sort the tokens into two groups; non-parse errors and parse errors |
| var expectedNonErrors = expectedTokens.where((t) => t != "ParseError"); |
| var receivedNonErrors = receivedTokens.where((t) => t != "ParseError"); |
| |
| expect(receivedNonErrors, equals(expectedNonErrors), reason: message); |
| if (!ignoreErrors) { |
| var expectedParseErrors = expectedTokens.where((t) => t == "ParseError"); |
| var receivedParseErrors = receivedTokens.where((t) => t == "ParseError"); |
| expect(receivedParseErrors, equals(expectedParseErrors), reason: message); |
| } |
| } |
| } |
| |
| void runTokenizerTest(Map testInfo) { |
| // XXX - move this out into the setup function |
| // concatenate all consecutive character tokens into a single token |
| if (testInfo.containsKey('doubleEscaped')) { |
| testInfo = unescape(testInfo); |
| } |
| |
| var expected = concatenateCharacterTokens(testInfo['output']); |
| if (!testInfo.containsKey('lastStartTag')) { |
| testInfo['lastStartTag'] = null; |
| } |
| var parser = new TokenizerTestParser( |
| testInfo['initialState'], testInfo['lastStartTag']); |
| var tokens = parser.parse(testInfo['input']); |
| tokens = concatenateCharacterTokens(tokens); |
| var received = normalizeTokens(tokens); |
| var errorMsg = [ |
| "\n\nInitial state:", |
| testInfo['initialState'], |
| "\nInput:", |
| testInfo['input'], |
| "\nExpected:", |
| expected, |
| "\nreceived:", |
| tokens |
| ].map((s) => '$s').join('\n'); |
| var ignoreErrorOrder = testInfo['ignoreErrorOrder']; |
| if (ignoreErrorOrder == null) ignoreErrorOrder = false; |
| |
| expectTokensMatch(expected, received, ignoreErrorOrder, true, errorMsg); |
| } |
| |
| Map unescape(Map testInfo) { |
| // TODO(sigmundch,jmesserly): we currently use JSON.decode to unescape the |
| // unicode characters in the string, we should use a decoding that works with |
| // any control characters. |
| decode(inp) => inp == '\u0000' ? inp : JSON.decode('"$inp"'); |
| |
| testInfo["input"] = decode(testInfo["input"]); |
| for (var token in testInfo["output"]) { |
| if (token == "ParseError") { |
| continue; |
| } else { |
| token[1] = decode(token[1]); |
| if (token.length > 2) { |
| for (var pair in token[2]) { |
| var key = pair[0]; |
| var value = pair[1]; |
| token[2].remove(key); |
| token[2][decode(key)] = decode(value); |
| } |
| } |
| } |
| } |
| return testInfo; |
| } |
| |
| String camelCase(String s) { |
| s = s.toLowerCase(); |
| var result = new StringBuffer(); |
| for (var match in new RegExp(r"\W+(\w)(\w+)").allMatches(s)) { |
| if (result.length == 0) result.write(s.substring(0, match.start)); |
| result.write(match.group(1).toUpperCase()); |
| result.write(match.group(2)); |
| } |
| return result.toString(); |
| } |
| |
| void main() { |
| for (var path in getDataFiles('tokenizer')) { |
| if (!path.endsWith('.test')) continue; |
| |
| var text = new File(path).readAsStringSync(); |
| var tests = JSON.decode(text); |
| var testName = pathos.basenameWithoutExtension(path); |
| var testList = tests['tests']; |
| if (testList == null) continue; |
| |
| group(testName, () { |
| for (int index = 0; index < testList.length; index++) { |
| final testInfo = testList[index]; |
| |
| testInfo.putIfAbsent("initialStates", () => ["Data state"]); |
| for (var initialState in testInfo["initialStates"]) { |
| test(testInfo["description"], () { |
| testInfo["initialState"] = camelCase(initialState); |
| runTokenizerTest(testInfo); |
| }); |
| } |
| } |
| }); |
| } |
| } |