Implement semantic tokens for regular comments
This commit is contained in:
Luuk de Gram
parent
ac8a00342e
commit
e2f4bbf2f3
@ -1785,8 +1785,19 @@ fn getDocumentSymbolsInternal(allocator: *std.mem.Allocator, tree: ast.Tree, nod
|
||||
if (name.len == 0)
|
||||
return;
|
||||
|
||||
const start_loc = context.prev_loc.add(try offsets.tokenRelativeLocation(tree, context.prev_loc.offset, tree.firstToken(node), context.encoding));
|
||||
const end_loc = start_loc.add(try offsets.tokenRelativeLocation(tree, start_loc.offset, tree.lastToken(node), context.encoding));
|
||||
const starts = tree.tokens.items(.start);
|
||||
const start_loc = context.prev_loc.add(try offsets.tokenRelativeLocation(
|
||||
tree,
|
||||
context.prev_loc.offset,
|
||||
starts[tree.firstToken(node)],
|
||||
context.encoding,
|
||||
));
|
||||
const end_loc = start_loc.add(try offsets.tokenRelativeLocation(
|
||||
tree,
|
||||
start_loc.offset,
|
||||
starts[tree.lastToken(node)],
|
||||
context.encoding,
|
||||
));
|
||||
context.prev_loc = end_loc;
|
||||
const range = types.Range{
|
||||
.start = .{
|
||||
@ -1908,7 +1919,7 @@ pub const DeclWithHandle = struct {
|
||||
|
||||
pub fn location(self: DeclWithHandle, encoding: offsets.Encoding) !offsets.TokenLocation {
|
||||
const tree = self.handle.tree;
|
||||
return try offsets.tokenRelativeLocation(tree, 0, self.nameToken(), encoding);
|
||||
return try offsets.tokenRelativeLocation(tree, 0, tree.tokens.items(.start)[self.nameToken()], encoding);
|
||||
}
|
||||
|
||||
fn isPublic(self: DeclWithHandle) bool {
|
||||
|
||||
@ -28,4 +28,4 @@ operator_completions: bool = true,
|
||||
|
||||
/// Skips references to std. This will improve lookup speeds.
|
||||
/// Going to definition however will continue to work
|
||||
skip_std_references: bool = true,
|
||||
skip_std_references: bool = false,
|
||||
|
||||
@ -554,7 +554,7 @@ fn gotoDefinitionSymbol(id: types.RequestId, arena: *std.heap.ArenaAllocator, de
|
||||
|
||||
const name_token = analysis.getDeclNameToken(handle.tree, node) orelse
|
||||
return try respondGeneric(id, null_result_response);
|
||||
break :block offsets.tokenRelativeLocation(handle.tree, 0, name_token, offset_encoding) catch return;
|
||||
break :block offsets.tokenRelativeLocation(handle.tree, 0, handle.tree.tokens.items(.start)[name_token], offset_encoding) catch return;
|
||||
},
|
||||
else => decl_handle.location(offset_encoding) catch return,
|
||||
};
|
||||
|
||||
@ -71,8 +71,8 @@ pub const TokenLocation = struct {
|
||||
}
|
||||
};
|
||||
|
||||
pub fn tokenRelativeLocation(tree: ast.Tree, start_index: usize, token: ast.TokenIndex, encoding: Encoding) !TokenLocation {
|
||||
const start = tree.tokens.items(.start)[token];
|
||||
pub fn tokenRelativeLocation(tree: ast.Tree, start_index: usize, next_token_index: usize, encoding: Encoding) !TokenLocation {
|
||||
const start = next_token_index;
|
||||
|
||||
var loc = TokenLocation{
|
||||
.line = 0,
|
||||
|
||||
@ -14,7 +14,7 @@ fn tokenReference(
|
||||
context: anytype,
|
||||
comptime handler: anytype,
|
||||
) !void {
|
||||
const loc = offsets.tokenRelativeLocation(handle.tree, 0, tok, encoding) catch return;
|
||||
const loc = offsets.tokenRelativeLocation(handle.tree, 0, handle.tree.tokens.items(.start)[tok], encoding) catch return;
|
||||
try handler(context, types.Location{
|
||||
.uri = handle.uri(),
|
||||
.range = .{
|
||||
|
||||
@ -48,11 +48,21 @@ pub const TokenModifiers = packed struct {
|
||||
}
|
||||
};
|
||||
|
||||
const Comment = struct {
|
||||
/// Length of the comment
|
||||
length: u32,
|
||||
/// Source index of the comment
|
||||
start: u32,
|
||||
};
|
||||
|
||||
const CommentList = std.ArrayList(Comment);
|
||||
|
||||
const Builder = struct {
|
||||
handle: *DocumentStore.Handle,
|
||||
current_token: ?ast.TokenIndex,
|
||||
arr: std.ArrayList(u32),
|
||||
encoding: offsets.Encoding,
|
||||
comments: CommentList,
|
||||
|
||||
fn init(allocator: *std.mem.Allocator, handle: *DocumentStore.Handle, encoding: offsets.Encoding) Builder {
|
||||
return Builder{
|
||||
@ -60,6 +70,7 @@ const Builder = struct {
|
||||
.current_token = null,
|
||||
.arr = std.ArrayList(u32).init(allocator),
|
||||
.encoding = encoding,
|
||||
.comments = CommentList.init(allocator),
|
||||
};
|
||||
}
|
||||
|
||||
@ -73,7 +84,21 @@ const Builder = struct {
|
||||
if (start_idx > starts[token])
|
||||
return;
|
||||
|
||||
const delta_loc = offsets.tokenRelativeLocation(self.handle.tree, start_idx, token, self.encoding) catch return;
|
||||
const delta_loc = if (self.findCommentBetween(start_idx, starts[token])) |comment| blk: {
|
||||
const old_loc = self.handle.tree.tokenLocation(0, self.current_token orelse 0);
|
||||
const comment_delta = offsets.tokenRelativeLocation(self.handle.tree, start_idx, comment.start, self.encoding) catch return;
|
||||
|
||||
try self.arr.appendSlice(&[_]u32{
|
||||
@truncate(u32, comment_delta.line),
|
||||
@truncate(u32, comment_delta.column),
|
||||
comment.length,
|
||||
@enumToInt(TokenType.comment),
|
||||
0,
|
||||
});
|
||||
|
||||
break :blk offsets.tokenRelativeLocation(self.handle.tree, comment.start, starts[token], self.encoding) catch return;
|
||||
} else offsets.tokenRelativeLocation(self.handle.tree, start_idx, starts[token], self.encoding) catch return;
|
||||
|
||||
try self.arr.appendSlice(&[_]u32{
|
||||
@truncate(u32, delta_loc.line),
|
||||
@truncate(u32, delta_loc.column),
|
||||
@ -87,6 +112,15 @@ const Builder = struct {
|
||||
fn toOwnedSlice(self: *Builder) []u32 {
|
||||
return self.arr.toOwnedSlice();
|
||||
}
|
||||
|
||||
/// Based on a given start and end index, returns a `Comment` between the positions
|
||||
/// Returns `null` if none was fone
|
||||
fn findCommentBetween(self: Builder, from: u32, to: u32) ?Comment {
|
||||
return for (self.comments.items) |comment| {
|
||||
if (comment.start > from and comment.start < to)
|
||||
break comment;
|
||||
} else null;
|
||||
}
|
||||
};
|
||||
|
||||
fn writeToken(
|
||||
@ -1011,8 +1045,12 @@ fn writeNodeTokens(
|
||||
pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentStore, handle: *DocumentStore.Handle, encoding: offsets.Encoding) ![]u32 {
|
||||
var builder = Builder.init(arena.child_allocator, handle, encoding);
|
||||
|
||||
// as line comments are not nodes, we parse the text then generate the tokens for them
|
||||
try findComments(&builder, handle.tree.source, encoding);
|
||||
|
||||
// reverse the ast from the root declarations
|
||||
var gap_highlighter = GapHighlighter.init(&builder, 0);
|
||||
|
||||
var buf: [2]ast.Node.Index = undefined;
|
||||
for (analysis.declMembers(handle.tree, .root, 0, &buf)) |child| {
|
||||
try gap_highlighter.next(child);
|
||||
@ -1020,7 +1058,57 @@ pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentS
|
||||
}
|
||||
|
||||
try gap_highlighter.end(@truncate(u32, handle.tree.tokens.len) - 1);
|
||||
// pass root node, which always has index '0'
|
||||
// try writeNodeTokens(&builder, arena, store, 0);
|
||||
|
||||
return builder.toOwnedSlice();
|
||||
}
|
||||
|
||||
/// As the AST does not contain nodes for comments
|
||||
/// this will parse through the entire file to search for comments
|
||||
/// and generate semantic tokens for them
|
||||
fn findComments(builder: *Builder, source: []const u8, encoding: offsets.Encoding) !void {
|
||||
var state: enum { none, comment, doc_comment } = .none;
|
||||
|
||||
var prev: u8 = 0;
|
||||
var start: usize = 0;
|
||||
for (source) |c, i| {
|
||||
if (state == .comment and c == '/') {
|
||||
state = .none;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (state == .none and c == '/' and prev == '/') {
|
||||
state = .comment;
|
||||
start = i - 1;
|
||||
}
|
||||
|
||||
if (c == '\n') {
|
||||
if (state == .comment) {
|
||||
state = .none;
|
||||
|
||||
const len = if (encoding == .utf8)
|
||||
i - start
|
||||
else blk: {
|
||||
var index: usize = start;
|
||||
var utf16_len: usize = 0;
|
||||
while (index < i) {
|
||||
const n = std.unicode.utf8ByteSequenceLength(source[index]) catch unreachable;
|
||||
const codepoint = std.unicode.utf8Decode(source[index .. index + n]) catch unreachable;
|
||||
if (codepoint < 0x10000) {
|
||||
utf16_len += 1;
|
||||
} else {
|
||||
utf16_len += 2;
|
||||
}
|
||||
index += n;
|
||||
}
|
||||
break :blk utf16_len;
|
||||
};
|
||||
|
||||
try builder.comments.append(.{
|
||||
.length = @truncate(u32, len),
|
||||
.start = @truncate(u32, start),
|
||||
});
|
||||
}
|
||||
}
|
||||
prev = c;
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user