diff --git a/src/analysis.zig b/src/analysis.zig index a061294..f25ba75 100644 --- a/src/analysis.zig +++ b/src/analysis.zig @@ -1785,8 +1785,19 @@ fn getDocumentSymbolsInternal(allocator: *std.mem.Allocator, tree: ast.Tree, nod if (name.len == 0) return; - const start_loc = context.prev_loc.add(try offsets.tokenRelativeLocation(tree, context.prev_loc.offset, tree.firstToken(node), context.encoding)); - const end_loc = start_loc.add(try offsets.tokenRelativeLocation(tree, start_loc.offset, tree.lastToken(node), context.encoding)); + const starts = tree.tokens.items(.start); + const start_loc = context.prev_loc.add(try offsets.tokenRelativeLocation( + tree, + context.prev_loc.offset, + starts[tree.firstToken(node)], + context.encoding, + )); + const end_loc = start_loc.add(try offsets.tokenRelativeLocation( + tree, + start_loc.offset, + starts[tree.lastToken(node)], + context.encoding, + )); context.prev_loc = end_loc; const range = types.Range{ .start = .{ @@ -1908,7 +1919,7 @@ pub const DeclWithHandle = struct { pub fn location(self: DeclWithHandle, encoding: offsets.Encoding) !offsets.TokenLocation { const tree = self.handle.tree; - return try offsets.tokenRelativeLocation(tree, 0, self.nameToken(), encoding); + return try offsets.tokenRelativeLocation(tree, 0, tree.tokens.items(.start)[self.nameToken()], encoding); } fn isPublic(self: DeclWithHandle) bool { diff --git a/src/config.zig b/src/config.zig index 346aa7d..fe224e2 100644 --- a/src/config.zig +++ b/src/config.zig @@ -28,4 +28,4 @@ operator_completions: bool = true, /// Skips references to std. This will improve lookup speeds. /// Going to definition however will continue to work -skip_std_references: bool = true, +skip_std_references: bool = false, diff --git a/src/main.zig b/src/main.zig index 11c12b5..89b3b1d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -554,7 +554,7 @@ fn gotoDefinitionSymbol(id: types.RequestId, arena: *std.heap.ArenaAllocator, de const name_token = analysis.getDeclNameToken(handle.tree, node) orelse return try respondGeneric(id, null_result_response); - break :block offsets.tokenRelativeLocation(handle.tree, 0, name_token, offset_encoding) catch return; + break :block offsets.tokenRelativeLocation(handle.tree, 0, handle.tree.tokens.items(.start)[name_token], offset_encoding) catch return; }, else => decl_handle.location(offset_encoding) catch return, }; diff --git a/src/offsets.zig b/src/offsets.zig index 418fe2f..8f38f56 100644 --- a/src/offsets.zig +++ b/src/offsets.zig @@ -71,8 +71,8 @@ pub const TokenLocation = struct { } }; -pub fn tokenRelativeLocation(tree: ast.Tree, start_index: usize, token: ast.TokenIndex, encoding: Encoding) !TokenLocation { - const start = tree.tokens.items(.start)[token]; +pub fn tokenRelativeLocation(tree: ast.Tree, start_index: usize, next_token_index: usize, encoding: Encoding) !TokenLocation { + const start = next_token_index; var loc = TokenLocation{ .line = 0, diff --git a/src/references.zig b/src/references.zig index b77749f..6c98824 100644 --- a/src/references.zig +++ b/src/references.zig @@ -14,7 +14,7 @@ fn tokenReference( context: anytype, comptime handler: anytype, ) !void { - const loc = offsets.tokenRelativeLocation(handle.tree, 0, tok, encoding) catch return; + const loc = offsets.tokenRelativeLocation(handle.tree, 0, handle.tree.tokens.items(.start)[tok], encoding) catch return; try handler(context, types.Location{ .uri = handle.uri(), .range = .{ diff --git a/src/semantic_tokens.zig b/src/semantic_tokens.zig index 87eaed9..a59eabd 100644 --- a/src/semantic_tokens.zig +++ b/src/semantic_tokens.zig @@ -48,11 +48,21 @@ pub const TokenModifiers = packed struct { } }; +const Comment = struct { + /// Length of the comment + length: u32, + /// Source index of the comment + start: u32, +}; + +const CommentList = std.ArrayList(Comment); + const Builder = struct { handle: *DocumentStore.Handle, current_token: ?ast.TokenIndex, arr: std.ArrayList(u32), encoding: offsets.Encoding, + comments: CommentList, fn init(allocator: *std.mem.Allocator, handle: *DocumentStore.Handle, encoding: offsets.Encoding) Builder { return Builder{ @@ -60,6 +70,7 @@ const Builder = struct { .current_token = null, .arr = std.ArrayList(u32).init(allocator), .encoding = encoding, + .comments = CommentList.init(allocator), }; } @@ -73,7 +84,21 @@ const Builder = struct { if (start_idx > starts[token]) return; - const delta_loc = offsets.tokenRelativeLocation(self.handle.tree, start_idx, token, self.encoding) catch return; + const delta_loc = if (self.findCommentBetween(start_idx, starts[token])) |comment| blk: { + const old_loc = self.handle.tree.tokenLocation(0, self.current_token orelse 0); + const comment_delta = offsets.tokenRelativeLocation(self.handle.tree, start_idx, comment.start, self.encoding) catch return; + + try self.arr.appendSlice(&[_]u32{ + @truncate(u32, comment_delta.line), + @truncate(u32, comment_delta.column), + comment.length, + @enumToInt(TokenType.comment), + 0, + }); + + break :blk offsets.tokenRelativeLocation(self.handle.tree, comment.start, starts[token], self.encoding) catch return; + } else offsets.tokenRelativeLocation(self.handle.tree, start_idx, starts[token], self.encoding) catch return; + try self.arr.appendSlice(&[_]u32{ @truncate(u32, delta_loc.line), @truncate(u32, delta_loc.column), @@ -87,6 +112,15 @@ const Builder = struct { fn toOwnedSlice(self: *Builder) []u32 { return self.arr.toOwnedSlice(); } + + /// Based on a given start and end index, returns a `Comment` between the positions + /// Returns `null` if none was fone + fn findCommentBetween(self: Builder, from: u32, to: u32) ?Comment { + return for (self.comments.items) |comment| { + if (comment.start > from and comment.start < to) + break comment; + } else null; + } }; fn writeToken( @@ -1011,8 +1045,12 @@ fn writeNodeTokens( pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentStore, handle: *DocumentStore.Handle, encoding: offsets.Encoding) ![]u32 { var builder = Builder.init(arena.child_allocator, handle, encoding); + // as line comments are not nodes, we parse the text then generate the tokens for them + try findComments(&builder, handle.tree.source, encoding); + // reverse the ast from the root declarations var gap_highlighter = GapHighlighter.init(&builder, 0); + var buf: [2]ast.Node.Index = undefined; for (analysis.declMembers(handle.tree, .root, 0, &buf)) |child| { try gap_highlighter.next(child); @@ -1020,7 +1058,57 @@ pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentS } try gap_highlighter.end(@truncate(u32, handle.tree.tokens.len) - 1); - // pass root node, which always has index '0' - // try writeNodeTokens(&builder, arena, store, 0); + return builder.toOwnedSlice(); } + +/// As the AST does not contain nodes for comments +/// this will parse through the entire file to search for comments +/// and generate semantic tokens for them +fn findComments(builder: *Builder, source: []const u8, encoding: offsets.Encoding) !void { + var state: enum { none, comment, doc_comment } = .none; + + var prev: u8 = 0; + var start: usize = 0; + for (source) |c, i| { + if (state == .comment and c == '/') { + state = .none; + continue; + } + + if (state == .none and c == '/' and prev == '/') { + state = .comment; + start = i - 1; + } + + if (c == '\n') { + if (state == .comment) { + state = .none; + + const len = if (encoding == .utf8) + i - start + else blk: { + var index: usize = start; + var utf16_len: usize = 0; + while (index < i) { + const n = std.unicode.utf8ByteSequenceLength(source[index]) catch unreachable; + const codepoint = std.unicode.utf8Decode(source[index .. index + n]) catch unreachable; + if (codepoint < 0x10000) { + utf16_len += 1; + } else { + utf16_len += 2; + } + index += n; + } + break :blk utf16_len; + }; + + try builder.comments.append(.{ + .length = @truncate(u32, len), + .start = @truncate(u32, start), + }); + } + } + prev = c; + } +}