blob: 3b9978747f0aade976e44386ebfa63b5b18da2e6 [file] [log] [blame]
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'package:analysis_server/lsp_protocol/protocol_generated.dart';
import 'package:analysis_server/src/lsp/constants.dart';
import 'package:analysis_server/src/lsp/semantic_tokens/legend.dart';
import 'package:analysis_server/src/protocol/protocol_internal.dart';
import 'package:analyzer_plugin/protocol/protocol_common.dart' as plugin;
import 'package:analyzer_plugin/protocol/protocol_generated.dart' as plugin;
import 'package:test/test.dart';
import 'package:test_reflective_loader/test_reflective_loader.dart';
import 'server_abstract.dart';
void main() {
defineReflectiveSuite(() {
defineReflectiveTests(SemanticTokensTest);
});
}
@reflectiveTest
class SemanticTokensTest extends AbstractLspAnalysisServerTest {
/// Decode tokens according to the LSP spec and pair with relevant file contents.
List<_Token> decodeSemanticTokens(String content, SemanticTokens tokens) {
final contentLines = content.split('\n').map((line) => '$line\n').toList();
final results = <_Token>[];
var lastLine = 0;
var lastColumn = 0;
for (var i = 0; i < tokens.data.length; i += 5) {
final lineDelta = tokens.data[i];
final columnDelta = tokens.data[i + 1];
final length = tokens.data[i + 2];
final tokenTypeIndex = tokens.data[i + 3];
final modifierBitmask = tokens.data[i + 4];
// Calculate the actual line/col from the deltas.
final line = lastLine + lineDelta;
final column = lineDelta == 0 ? lastColumn + columnDelta : columnDelta;
final tokenContent =
contentLines[line].substring(column, column + length);
results.add(_Token(
tokenContent,
semanticTokenLegend.typeForIndex(tokenTypeIndex),
semanticTokenLegend.modifiersForBitmask(modifierBitmask),
));
lastLine = line;
lastColumn = column;
}
return results;
}
Future<void> test_annotation() async {
final content = '''
import 'other_file.dart' as other;
@a
@A()
@A.n()
@B(A())
@other.C()
@other.C.n()
void foo() {}
class A {
const A();
const A.n();
}
const a = A();
class B {
final A a;
const B(this.a);
}
''';
final otherContent = '''
class C {
const C();
const C.n();
}
''';
final expectedStart = [
_Token('import', SemanticTokenTypes.keyword),
_Token("'other_file.dart'", SemanticTokenTypes.string),
_Token('as', SemanticTokenTypes.keyword),
_Token('other', SemanticTokenTypes.variable,
[CustomSemanticTokenModifiers.importPrefix]),
_Token('@', CustomSemanticTokenTypes.annotation),
_Token('a', SemanticTokenTypes.property,
[CustomSemanticTokenModifiers.annotation]),
_Token('@', CustomSemanticTokenTypes.annotation),
_Token('A', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.annotation]),
_Token('(', CustomSemanticTokenTypes.annotation),
_Token(')', CustomSemanticTokenTypes.annotation),
_Token('@', CustomSemanticTokenTypes.annotation),
_Token('A', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.annotation]),
_Token('.', CustomSemanticTokenTypes.annotation),
_Token('n', SemanticTokenTypes.method, [
CustomSemanticTokenModifiers.constructor,
CustomSemanticTokenModifiers.annotation
]),
_Token('(', CustomSemanticTokenTypes.annotation),
_Token(')', CustomSemanticTokenTypes.annotation),
_Token('@', CustomSemanticTokenTypes.annotation),
_Token('B', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.annotation]),
_Token('(', CustomSemanticTokenTypes.annotation),
_Token('A', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token(')', CustomSemanticTokenTypes.annotation),
_Token('@', CustomSemanticTokenTypes.annotation),
_Token('other', SemanticTokenTypes.variable,
[CustomSemanticTokenModifiers.importPrefix]),
_Token('.', CustomSemanticTokenTypes.annotation),
_Token('C', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.annotation]),
_Token('(', CustomSemanticTokenTypes.annotation),
_Token(')', CustomSemanticTokenTypes.annotation),
_Token('@', CustomSemanticTokenTypes.annotation),
_Token('other', SemanticTokenTypes.variable,
[CustomSemanticTokenModifiers.importPrefix]),
_Token('.', CustomSemanticTokenTypes.annotation),
_Token('C', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.annotation]),
_Token('.', CustomSemanticTokenTypes.annotation),
_Token('n', SemanticTokenTypes.method, [
CustomSemanticTokenModifiers.constructor,
CustomSemanticTokenModifiers.annotation
]),
_Token('(', CustomSemanticTokenTypes.annotation),
_Token(')', CustomSemanticTokenTypes.annotation),
_Token('void', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.void_]),
_Token('foo', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static])
];
final otherFilePath = join(projectFolderPath, 'lib', 'other_file.dart');
final otherFileUri = Uri.file(otherFilePath);
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
await openFile(otherFileUri, withoutMarkers(otherContent));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(
// Only check the first expectedStart.length items since the test code
// is mostly unrelated to the annotations.
decoded.sublist(0, expectedStart.length),
equals(expectedStart),
);
}
Future<void> test_class() async {
final content = '''
/// class docs
class MyClass<T> {
// class comment
}
// Trailing comment
''';
final expected = [
_Token('/// class docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('T', SemanticTokenTypes.typeParameter),
_Token('// class comment', SemanticTokenTypes.comment),
_Token('// Trailing comment', SemanticTokenTypes.comment),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_class_constructors() async {
final content = '''
class MyClass {
MyClass();
MyClass.named();
factory MyClass.factory() => MyClass();
}
final a = MyClass();
final b = MyClass.named();
final c = MyClass.factory();
final d = MyClass.named;
''';
final expected = [
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('named', SemanticTokenTypes.method,
[CustomSemanticTokenModifiers.constructor]),
_Token('factory', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('factory', SemanticTokenTypes.method,
[CustomSemanticTokenModifiers.constructor]),
_Token('MyClass', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('final', SemanticTokenTypes.keyword),
_Token('a', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('final', SemanticTokenTypes.keyword),
_Token('b', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('named', SemanticTokenTypes.method,
[CustomSemanticTokenModifiers.constructor]),
_Token('final', SemanticTokenTypes.keyword),
_Token('c', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('factory', SemanticTokenTypes.method,
[CustomSemanticTokenModifiers.constructor]),
_Token('final', SemanticTokenTypes.keyword),
_Token('d', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('named', SemanticTokenTypes.method,
[CustomSemanticTokenModifiers.constructor]),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_class_fields() async {
final content = '''
class MyClass {
/// field docs
String myField = 'FieldVal';
/// static field docs
static String myStaticField = 'StaticFieldVal';
}
main() {
final a = MyClass();
print(a.myField);
MyClass.myStaticField = 'a';
}
''';
final expected = [
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('/// field docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('String', SemanticTokenTypes.class_),
_Token('myField', SemanticTokenTypes.property, [
SemanticTokenModifiers.declaration,
CustomSemanticTokenModifiers.instance
]),
_Token("'FieldVal'", SemanticTokenTypes.string),
_Token('/// static field docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('static', SemanticTokenTypes.keyword),
_Token('String', SemanticTokenTypes.class_),
_Token('myStaticField', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token("'StaticFieldVal'", SemanticTokenTypes.string),
_Token('main', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('final', SemanticTokenTypes.keyword),
_Token('a', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('print', SemanticTokenTypes.function),
_Token('a', SemanticTokenTypes.variable),
_Token('myField', SemanticTokenTypes.property,
[CustomSemanticTokenModifiers.instance]),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('myStaticField', SemanticTokenTypes.property,
[SemanticTokenModifiers.static]),
_Token("'a'", SemanticTokenTypes.string),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_class_getterSetter() async {
final content = '''
class MyClass {
/// getter docs
String get myGetter => 'GetterVal';
/// setter docs
set mySetter(String v) {};
/// static getter docs
static String get myStaticGetter => 'StaticGetterVal';
/// static setter docs
static set myStaticSetter(String staticV) {};
}
main() {
final a = MyClass();
print(a.myGetter);
a.mySetter = 'a';
}
''';
final expected = [
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('/// getter docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('String', SemanticTokenTypes.class_),
_Token('get', SemanticTokenTypes.keyword),
_Token('myGetter', SemanticTokenTypes.property, [
SemanticTokenModifiers.declaration,
CustomSemanticTokenModifiers.instance
]),
_Token("'GetterVal'", SemanticTokenTypes.string),
_Token('/// setter docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('set', SemanticTokenTypes.keyword),
_Token('mySetter', SemanticTokenTypes.property, [
SemanticTokenModifiers.declaration,
CustomSemanticTokenModifiers.instance
]),
_Token('String', SemanticTokenTypes.class_),
_Token('v', SemanticTokenTypes.parameter,
[SemanticTokenModifiers.declaration]),
_Token('/// static getter docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('static', SemanticTokenTypes.keyword),
_Token('String', SemanticTokenTypes.class_),
_Token('get', SemanticTokenTypes.keyword),
_Token('myStaticGetter', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token("'StaticGetterVal'", SemanticTokenTypes.string),
_Token('/// static setter docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('static', SemanticTokenTypes.keyword),
_Token('set', SemanticTokenTypes.keyword),
_Token('myStaticSetter', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('String', SemanticTokenTypes.class_),
_Token('staticV', SemanticTokenTypes.parameter,
[SemanticTokenModifiers.declaration]),
_Token('main', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('final', SemanticTokenTypes.keyword),
_Token('a', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('print', SemanticTokenTypes.function),
_Token('a', SemanticTokenTypes.variable),
_Token('myGetter', SemanticTokenTypes.property,
[CustomSemanticTokenModifiers.instance]),
_Token('a', SemanticTokenTypes.variable),
_Token('mySetter', SemanticTokenTypes.property,
[CustomSemanticTokenModifiers.instance]),
_Token("'a'", SemanticTokenTypes.string),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_class_method() async {
final content = '''
class MyClass {
/// method docs
@override
void myMethod() {}
/// static method docs
static void myStaticMethod() {
// static method comment
}
}
main() {
final a = MyClass();
a.myMethod();
MyClass.myStaticMethod();
final b = a.myMethod;
final c = MyClass.myStaticMethod;
}
''';
final expected = [
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('/// method docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('@', CustomSemanticTokenTypes.annotation),
_Token('override', SemanticTokenTypes.property,
[CustomSemanticTokenModifiers.annotation]),
_Token('void', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.void_]),
_Token('myMethod', SemanticTokenTypes.method, [
SemanticTokenModifiers.declaration,
CustomSemanticTokenModifiers.instance
]),
_Token('/// static method docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('static', SemanticTokenTypes.keyword),
_Token('void', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.void_]),
_Token('myStaticMethod', SemanticTokenTypes.method,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('// static method comment', SemanticTokenTypes.comment),
_Token('main', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('final', SemanticTokenTypes.keyword),
_Token('a', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('a', SemanticTokenTypes.variable),
_Token('myMethod', SemanticTokenTypes.method,
[CustomSemanticTokenModifiers.instance]),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('myStaticMethod', SemanticTokenTypes.method,
[SemanticTokenModifiers.static]),
_Token('final', SemanticTokenTypes.keyword),
_Token('b', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('a', SemanticTokenTypes.variable),
_Token('myMethod', SemanticTokenTypes.method,
[CustomSemanticTokenModifiers.instance]),
_Token('final', SemanticTokenTypes.keyword),
_Token('c', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('myStaticMethod', SemanticTokenTypes.method,
[SemanticTokenModifiers.static]),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_dartdoc() async {
final content = '''
/// before [aaa] after
class MyClass {
String aaa;
}
/// before [bbb] after
int double(int bbb) => bbb * 2;
''';
final expected = [
_Token('/// before [', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('aaa', SemanticTokenTypes.property,
[CustomSemanticTokenModifiers.instance]),
_Token('] after', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('String', SemanticTokenTypes.class_),
_Token('aaa', SemanticTokenTypes.property, [
SemanticTokenModifiers.declaration,
CustomSemanticTokenModifiers.instance
]),
_Token('/// before [', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('bbb', SemanticTokenTypes.parameter),
_Token('] after', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('int', SemanticTokenTypes.class_),
_Token('double', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('int', SemanticTokenTypes.class_),
_Token('bbb', SemanticTokenTypes.parameter,
[SemanticTokenModifiers.declaration]),
_Token('bbb', SemanticTokenTypes.parameter),
_Token('2', SemanticTokenTypes.number)
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_directives() async {
final content = '''
import 'package:flutter/material.dart';
export 'package:flutter/widgets.dart';
import '../file.dart'
if (dart.library.io) 'file_io.dart'
if (dart.library.html) 'file_html.dart';
library foo;
''';
final expected = [
_Token('import', SemanticTokenTypes.keyword),
_Token("'package:flutter/material.dart'", SemanticTokenTypes.string),
_Token('export', SemanticTokenTypes.keyword),
_Token("'package:flutter/widgets.dart'", SemanticTokenTypes.string),
_Token('import', SemanticTokenTypes.keyword),
_Token("'../file.dart'", SemanticTokenTypes.string),
_Token('if', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.control]),
_Token('dart', CustomSemanticTokenTypes.source),
_Token('library', CustomSemanticTokenTypes.source),
_Token('io', CustomSemanticTokenTypes.source),
_Token("'file_io.dart'", SemanticTokenTypes.string),
_Token('if', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.control]),
_Token('dart', CustomSemanticTokenTypes.source),
_Token('library', CustomSemanticTokenTypes.source),
_Token('html', CustomSemanticTokenTypes.source),
_Token("'file_html.dart'", SemanticTokenTypes.string),
_Token('library', SemanticTokenTypes.keyword),
_Token('foo', SemanticTokenTypes.namespace),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_extension() async {
final content = '''
extension A on String {}
''';
final expected = [
_Token('extension', SemanticTokenTypes.keyword),
_Token('A', SemanticTokenTypes.class_),
_Token('on', SemanticTokenTypes.keyword),
_Token('String', SemanticTokenTypes.class_)
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_fromPlugin() async {
final pluginAnalyzedFilePath = join(projectFolderPath, 'lib', 'foo.foo');
final pluginAnalyzedFileUri = Uri.file(pluginAnalyzedFilePath);
final content = 'CLASS STRING VARIABLE';
final expected = [
_Token('CLASS', SemanticTokenTypes.class_),
_Token('STRING', SemanticTokenTypes.string),
_Token('VARIABLE', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
];
await initialize();
await openFile(pluginAnalyzedFileUri, withoutMarkers(content));
final pluginResult = plugin.AnalysisHighlightsParams(
pluginAnalyzedFilePath,
[
plugin.HighlightRegion(plugin.HighlightRegionType.CLASS, 0, 5),
plugin.HighlightRegion(plugin.HighlightRegionType.LITERAL_STRING, 6, 6),
plugin.HighlightRegion(
plugin.HighlightRegionType.LOCAL_VARIABLE_DECLARATION, 13, 8),
],
);
configureTestPlugin(notification: pluginResult.toNotification());
final tokens = await getSemanticTokens(pluginAnalyzedFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_invalidSyntax() async {
final content = '''
/// class docs
class MyClass {
// class comment
}
this is not valid code.
/// class docs 2
class MyClass2 {
// class comment 2
}
''';
// Expect the correct tokens for the valid code before/after but don't
// check the tokens for the invalid code as there are no concrete
// expectations for them.
final expected1 = [
_Token('/// class docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('// class comment', SemanticTokenTypes.comment),
];
final expected2 = [
_Token('/// class docs 2', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass2', SemanticTokenTypes.class_),
_Token('// class comment 2', SemanticTokenTypes.comment),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
// Remove the tokens between the two expected sets.
decoded.removeRange(expected1.length, decoded.length - expected2.length);
expect(decoded, equals([...expected1, ...expected2]));
}
Future<void> test_keywords() async {
// "control" keywords should be tagged with a modifier so the client
// can color them differently to other keywords.
final content = r'''
void main() async {
var a = new Object();
await null;
if (false) {
print('test');
}
}
''';
final expected = [
_Token('void', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.void_]),
_Token('main', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('async', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.control]),
_Token('var', SemanticTokenTypes.keyword),
_Token('a', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('new', SemanticTokenTypes.keyword),
_Token('Object', SemanticTokenTypes.class_,
[CustomSemanticTokenModifiers.constructor]),
_Token('await', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.control]),
_Token('null', SemanticTokenTypes.keyword),
_Token('if', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.control]),
_Token('false', CustomSemanticTokenTypes.boolean),
_Token('print', SemanticTokenTypes.function),
_Token("'test'", SemanticTokenTypes.string),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_lastLine_code() async {
final content = 'String bar;';
final expected = [
_Token('String', SemanticTokenTypes.class_),
_Token('bar', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_lastLine_comment() async {
final content = '// Trailing comment';
final expected = [
_Token('// Trailing comment', SemanticTokenTypes.comment),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_lastLine_multilineComment() async {
final content = '''/**
* Trailing comment
*/''';
final expected = [
_Token('/**\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' * Trailing comment\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' */', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_local() async {
final content = '''
main() {
func(String a) => print(a);
final funcTearOff = func;
}
''';
final expected = [
_Token('main', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('func', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration]),
_Token('String', SemanticTokenTypes.class_),
_Token('a', SemanticTokenTypes.parameter,
[SemanticTokenModifiers.declaration]),
_Token('print', SemanticTokenTypes.function),
_Token('a', SemanticTokenTypes.parameter),
_Token('final', SemanticTokenTypes.keyword),
_Token('funcTearOff', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('func', SemanticTokenTypes.function),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_manyBools_bug() async {
// Similar to test_manyImports_sortBug, this code triggered inconsistent tokens
// for "false" because tokens were sorted incorrectly (because both boolean and
// keyword had the same offset and length, which is all that were sorted by).
final content = '''
class MyTestClass {
/// test
/// test
bool test1 = false;
/// test
/// test
bool test2 = false;
/// test
/// test
bool test3 = false;
/// test
/// test
bool test4 = false;
/// test
/// test
bool test5 = false;
/// test
/// test
bool test6 = false;
}
''';
final expected = [
_Token('class', SemanticTokenTypes.keyword),
_Token('MyTestClass', SemanticTokenTypes.class_),
for (var i = 1; i <= 6; i++) ...[
_Token('/// test', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('/// test', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('bool', SemanticTokenTypes.class_),
_Token('test$i', SemanticTokenTypes.property, [
SemanticTokenModifiers.declaration,
CustomSemanticTokenModifiers.instance
]),
_Token('false', CustomSemanticTokenTypes.boolean),
],
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_manyImports_sortBug() async {
// This test is for a bug where some "import" tokens would not be highlighted
// correctly. Imports are made up of a DIRECTIVE token that spans a
// BUILT_IN ("import") and LITERAL_STRING. The original code sorted by only
// offset when handling overlapping tokens, which for certain lists (such as
// the one created for the code below) would result in the BUILTIN coming before
// the DIRECTIVE, which resulted in the DIRECTIVE overwriting it.
final content = '''
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
import 'dart:async';
''';
final expected = [
for (var i = 0; i < 13; i++) ...[
_Token('import', SemanticTokenTypes.keyword),
_Token("'dart:async'", SemanticTokenTypes.string),
],
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_multilineRegions() async {
final content = '''
/**
* This is my class comment
*
* There are
* multiple lines
*/
class MyClass {}
''';
final expected = [
_Token('/**\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' * This is my class comment\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' *\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' * There are\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' * multiple lines\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' */', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_namedArguments() async {
final content = '''
f({String a}) {
f(a: a);
}
''';
final expected = [
_Token('f', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('String', SemanticTokenTypes.class_),
_Token('a', SemanticTokenTypes.parameter,
[SemanticTokenModifiers.declaration]),
_Token('f', SemanticTokenTypes.function),
_Token('a', SemanticTokenTypes.parameter,
[CustomSemanticTokenModifiers.label]),
_Token('a', SemanticTokenTypes.parameter),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_range() async {
final content = '''
/// class docs
class [[MyClass<T> {
// class comment
}]]
// Trailing comment
''';
final expected = [
_Token('MyClass', SemanticTokenTypes.class_),
_Token('T', SemanticTokenTypes.typeParameter),
_Token('// class comment', SemanticTokenTypes.comment),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens =
await getSemanticTokensRange(mainFileUri, rangeFromMarkers(content));
final decoded = decodeSemanticTokens(withoutMarkers(content), tokens);
expect(decoded, equals(expected));
}
Future<void> test_range_entireFile() async {
final content = '''[[
/// class docs
class MyClass<T> {
// class comment
}
// Trailing comment
]]''';
final expected = [
_Token('/// class docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('class', SemanticTokenTypes.keyword),
_Token('MyClass', SemanticTokenTypes.class_),
_Token('T', SemanticTokenTypes.typeParameter),
_Token('// class comment', SemanticTokenTypes.comment),
_Token('// Trailing comment', SemanticTokenTypes.comment),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens =
await getSemanticTokensRange(mainFileUri, rangeFromMarkers(content));
final decoded = decodeSemanticTokens(withoutMarkers(content), tokens);
expect(decoded, equals(expected));
}
Future<void> test_range_multilineRegions() async {
final content = '''
/**
* This is my class comment
*
* [[There are
* multiple lines
*/
class]] MyClass {}
''';
final expected = [
_Token(' * There are\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' * multiple lines\n', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token(' */', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('class', SemanticTokenTypes.keyword),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens =
await getSemanticTokensRange(mainFileUri, rangeFromMarkers(content));
final decoded = decodeSemanticTokens(withoutMarkers(content), tokens);
expect(decoded, equals(expected));
}
Future<void> test_strings() async {
final content = '''
String foo(String c) => c;
const string1 = 'test';
const string2 = 'test1 \$string1 test2 \${foo('a' + 'b')}';
const string3 = r'\$string1 \${string1.length}';
const string4 = \'\'\'
multi
line
string
\'\'\';
''';
final expected = [
_Token('String', SemanticTokenTypes.class_),
_Token('foo', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('String', SemanticTokenTypes.class_),
_Token('c', SemanticTokenTypes.parameter,
[SemanticTokenModifiers.declaration]),
_Token('c', SemanticTokenTypes.parameter),
_Token('const', SemanticTokenTypes.keyword),
_Token('string1', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token("'test'", SemanticTokenTypes.string),
_Token('const', SemanticTokenTypes.keyword),
_Token('string2', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token(r"'test1 ", SemanticTokenTypes.string),
_Token(r'$', CustomSemanticTokenTypes.source,
[CustomSemanticTokenModifiers.interpolation]),
_Token('string1', SemanticTokenTypes.property),
_Token(' test2 ', SemanticTokenTypes.string),
_Token(r'${', CustomSemanticTokenTypes.source,
[CustomSemanticTokenModifiers.interpolation]),
_Token('foo', SemanticTokenTypes.function),
_Token('(', CustomSemanticTokenTypes.source,
[CustomSemanticTokenModifiers.interpolation]),
_Token("'a'", SemanticTokenTypes.string),
_Token(' + ', CustomSemanticTokenTypes.source,
[CustomSemanticTokenModifiers.interpolation]),
_Token("'b'", SemanticTokenTypes.string),
_Token(')}', CustomSemanticTokenTypes.source,
[CustomSemanticTokenModifiers.interpolation]),
_Token("'", SemanticTokenTypes.string),
// string3 is raw and should be treated as a single string.
_Token('const', SemanticTokenTypes.keyword),
_Token('string3', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token(r"r'$string1 ${string1.length}'", SemanticTokenTypes.string),
_Token('const', SemanticTokenTypes.keyword),
_Token('string4', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token("'''\n", SemanticTokenTypes.string),
_Token('multi\n', SemanticTokenTypes.string),
_Token(' line\n', SemanticTokenTypes.string),
_Token(' string\n', SemanticTokenTypes.string),
_Token("'''", SemanticTokenTypes.string),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_strings_escape() async {
// The 9's in these strings are not part of the escapes (they make the
// strings too long).
final content = r'''
const string1 = 'it\'s escaped\\\n';
const string2 = 'hex \x12\x1299';
const string3 = 'unicode \u1234\u123499\u{123456}\u{12345699}';
''';
final expected = [
_Token('const', SemanticTokenTypes.keyword),
_Token('string1', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token("'it", SemanticTokenTypes.string),
_Token(r"\'", SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
_Token('s escaped', SemanticTokenTypes.string),
_Token(r'\\', SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
_Token(r'\n', SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
_Token(r"'", SemanticTokenTypes.string),
_Token('const', SemanticTokenTypes.keyword),
_Token('string2', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token("'hex ", SemanticTokenTypes.string),
_Token(r'\x12', SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
_Token(r'\x12', SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
// The 99 is not part of the escape
_Token("99'", SemanticTokenTypes.string),
_Token('const', SemanticTokenTypes.keyword),
_Token('string3', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token("'unicode ", SemanticTokenTypes.string),
_Token(r'\u1234', SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
_Token(r'\u1234', SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
// The 99 is not part of the escape
_Token('99', SemanticTokenTypes.string),
_Token(r'\u{123456}', SemanticTokenTypes.string,
[CustomSemanticTokenModifiers.escape]),
// The 99 makes this invalid so i's not an escape
_Token(r"\u{12345699}'", SemanticTokenTypes.string),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_topLevel() async {
final content = '''
/// strings docs
const strings = <String>["test", 'test', r'test', \'''test\'''];
/// func docs
func(String a) => print(a);
/// abc docs
bool get abc => true;
final funcTearOff = func;
void main() {
strings;
func;
abc;
funcTearOff;
}
''';
final expected = [
_Token('/// strings docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('const', SemanticTokenTypes.keyword),
_Token('strings', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token('String', SemanticTokenTypes.class_),
_Token('"test"', SemanticTokenTypes.string),
_Token("'test'", SemanticTokenTypes.string),
_Token("r'test'", SemanticTokenTypes.string),
_Token("'''test'''", SemanticTokenTypes.string),
_Token('/// func docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('func', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('String', SemanticTokenTypes.class_),
_Token('a', SemanticTokenTypes.parameter,
[SemanticTokenModifiers.declaration]),
_Token('print', SemanticTokenTypes.function),
_Token('a', SemanticTokenTypes.parameter),
_Token('/// abc docs', SemanticTokenTypes.comment,
[SemanticTokenModifiers.documentation]),
_Token('bool', SemanticTokenTypes.class_),
_Token('get', SemanticTokenTypes.keyword),
_Token('abc', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token('true', CustomSemanticTokenTypes.boolean),
_Token('final', SemanticTokenTypes.keyword),
_Token('funcTearOff', SemanticTokenTypes.property,
[SemanticTokenModifiers.declaration]),
_Token('func', SemanticTokenTypes.function),
_Token('void', SemanticTokenTypes.keyword,
[CustomSemanticTokenModifiers.void_]),
_Token('main', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('strings', SemanticTokenTypes.property),
_Token('func', SemanticTokenTypes.function),
_Token('abc', SemanticTokenTypes.property),
_Token('funcTearOff', SemanticTokenTypes.property),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
Future<void> test_unresolvedOrInvalid() async {
// Unresolved/invalid names should be marked as "source", which is used to
// mark up code the server thinks should be uncolored (without this, a
// clients other grammars would show through, losing the benefit from having
// resolved the code).
final content = '''
main() {
int a;
a.foo().bar.baz();
dynamic b;
b.foo().bar.baz();
}
''';
final expected = [
_Token('main', SemanticTokenTypes.function,
[SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
_Token('int', SemanticTokenTypes.class_),
_Token('a', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('a', SemanticTokenTypes.variable),
_Token('foo', CustomSemanticTokenTypes.source),
_Token('bar', CustomSemanticTokenTypes.source),
_Token('baz', CustomSemanticTokenTypes.source),
_Token('dynamic', SemanticTokenTypes.type),
_Token('b', SemanticTokenTypes.variable,
[SemanticTokenModifiers.declaration]),
_Token('b', SemanticTokenTypes.variable),
_Token('foo', CustomSemanticTokenTypes.source),
_Token('bar', CustomSemanticTokenTypes.source),
_Token('baz', CustomSemanticTokenTypes.source),
];
await initialize();
await openFile(mainFileUri, withoutMarkers(content));
final tokens = await getSemanticTokens(mainFileUri);
final decoded = decodeSemanticTokens(content, tokens);
expect(decoded, equals(expected));
}
}
class _Token {
final String content;
final SemanticTokenTypes type;
final List<SemanticTokenModifiers> modifiers;
_Token(this.content, this.type, [this.modifiers = const []]);
@override
int get hashCode => content.hashCode;
@override
bool operator ==(Object o) =>
o is _Token &&
o.content == content &&
o.type == type &&
listEqual(
// Treat nulls the same as empty lists for convenience when comparing.
o.modifiers,
modifiers,
(SemanticTokenModifiers a, SemanticTokenModifiers b) => a == b);
/// Outputs a text representation of the token in the form of constructor
/// args for easy copy/pasting into tests to update expectations.
@override
String toString() {
final modifiersString = modifiers.isEmpty
? ''
: ', [${modifiers.map((m) => 'SemanticTokenModifiers.$m').join(', ')}]';
return "('$content', SemanticTokenTypes.$type$modifiersString)";
}
}