Implement semantic tokens for regular comments

src/analysis.zig

@@ -1785,8 +1785,19 @@ fn getDocumentSymbolsInternal(allocator: *std.mem.Allocator, tree: ast.Tree, nod
     if (name.len == 0)
         return;
 
-    const start_loc = context.prev_loc.add(try offsets.tokenRelativeLocation(tree, context.prev_loc.offset, tree.firstToken(node), context.encoding));
+    const starts = tree.tokens.items(.start);
-    const end_loc = start_loc.add(try offsets.tokenRelativeLocation(tree, start_loc.offset, tree.lastToken(node), context.encoding));
+    const start_loc = context.prev_loc.add(try offsets.tokenRelativeLocation(
+        tree,
+        context.prev_loc.offset,
+        starts[tree.firstToken(node)],
+        context.encoding,
+    ));
+    const end_loc = start_loc.add(try offsets.tokenRelativeLocation(
+        tree,
+        start_loc.offset,
+        starts[tree.lastToken(node)],
+        context.encoding,
+    ));
     context.prev_loc = end_loc;
     const range = types.Range{
         .start = .{
@@ -1908,7 +1919,7 @@ pub const DeclWithHandle = struct {
 
     pub fn location(self: DeclWithHandle, encoding: offsets.Encoding) !offsets.TokenLocation {
         const tree = self.handle.tree;
-        return try offsets.tokenRelativeLocation(tree, 0, self.nameToken(), encoding);
+        return try offsets.tokenRelativeLocation(tree, 0, tree.tokens.items(.start)[self.nameToken()], encoding);
     }
 
     fn isPublic(self: DeclWithHandle) bool {

src/config.zig

@@ -28,4 +28,4 @@ operator_completions: bool = true,
 
 /// Skips references to std. This will improve lookup speeds.
 /// Going to definition however will continue to work
-skip_std_references: bool = true,
+skip_std_references: bool = false,

src/main.zig

@@ -554,7 +554,7 @@ fn gotoDefinitionSymbol(id: types.RequestId, arena: *std.heap.ArenaAllocator, de
 
             const name_token = analysis.getDeclNameToken(handle.tree, node) orelse
                 return try respondGeneric(id, null_result_response);
-            break :block offsets.tokenRelativeLocation(handle.tree, 0, name_token, offset_encoding) catch return;
+            break :block offsets.tokenRelativeLocation(handle.tree, 0, handle.tree.tokens.items(.start)[name_token], offset_encoding) catch return;
         },
         else => decl_handle.location(offset_encoding) catch return,
     };

src/offsets.zig

@@ -71,8 +71,8 @@ pub const TokenLocation = struct {
     }
 };
 
-pub fn tokenRelativeLocation(tree: ast.Tree, start_index: usize, token: ast.TokenIndex, encoding: Encoding) !TokenLocation {
+pub fn tokenRelativeLocation(tree: ast.Tree, start_index: usize, next_token_index: usize, encoding: Encoding) !TokenLocation {
-    const start = tree.tokens.items(.start)[token];
+    const start = next_token_index;
 
     var loc = TokenLocation{
         .line = 0,

src/references.zig

@@ -14,7 +14,7 @@ fn tokenReference(
     context: anytype,
     comptime handler: anytype,
 ) !void {
-    const loc = offsets.tokenRelativeLocation(handle.tree, 0, tok, encoding) catch return;
+    const loc = offsets.tokenRelativeLocation(handle.tree, 0, handle.tree.tokens.items(.start)[tok], encoding) catch return;
     try handler(context, types.Location{
         .uri = handle.uri(),
         .range = .{

src/semantic_tokens.zig

@@ -48,11 +48,21 @@ pub const TokenModifiers = packed struct {
     }
 };
 
+const Comment = struct {
+    /// Length of the comment
+    length: u32,
+    /// Source index of the comment
+    start: u32,
+};
+
+const CommentList = std.ArrayList(Comment);
+
 const Builder = struct {
     handle: *DocumentStore.Handle,
     current_token: ?ast.TokenIndex,
     arr: std.ArrayList(u32),
     encoding: offsets.Encoding,
+    comments: CommentList,
 
     fn init(allocator: *std.mem.Allocator, handle: *DocumentStore.Handle, encoding: offsets.Encoding) Builder {
         return Builder{
@@ -60,6 +70,7 @@ const Builder = struct {
             .current_token = null,
             .arr = std.ArrayList(u32).init(allocator),
             .encoding = encoding,
+            .comments = CommentList.init(allocator),
         };
     }
 
@@ -73,7 +84,21 @@ const Builder = struct {
         if (start_idx > starts[token])
             return;
 
-        const delta_loc = offsets.tokenRelativeLocation(self.handle.tree, start_idx, token, self.encoding) catch return;
+        const delta_loc = if (self.findCommentBetween(start_idx, starts[token])) |comment| blk: {
+            const old_loc = self.handle.tree.tokenLocation(0, self.current_token orelse 0);
+            const comment_delta = offsets.tokenRelativeLocation(self.handle.tree, start_idx, comment.start, self.encoding) catch return;
+
+            try self.arr.appendSlice(&[_]u32{
+                @truncate(u32, comment_delta.line),
+                @truncate(u32, comment_delta.column),
+                comment.length,
+                @enumToInt(TokenType.comment),
+                0,
+            });
+
+            break :blk offsets.tokenRelativeLocation(self.handle.tree, comment.start, starts[token], self.encoding) catch return;
+        } else offsets.tokenRelativeLocation(self.handle.tree, start_idx, starts[token], self.encoding) catch return;
+
         try self.arr.appendSlice(&[_]u32{
             @truncate(u32, delta_loc.line),
             @truncate(u32, delta_loc.column),
@@ -87,6 +112,15 @@ const Builder = struct {
     fn toOwnedSlice(self: *Builder) []u32 {
         return self.arr.toOwnedSlice();
     }
+
+    /// Based on a given start and end index, returns a `Comment` between the positions
+    /// Returns `null` if none was fone
+    fn findCommentBetween(self: Builder, from: u32, to: u32) ?Comment {
+        return for (self.comments.items) |comment| {
+            if (comment.start > from and comment.start < to)
+                break comment;
+        } else null;
+    }
 };
 
 fn writeToken(
@@ -1011,8 +1045,12 @@ fn writeNodeTokens(
 pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentStore, handle: *DocumentStore.Handle, encoding: offsets.Encoding) ![]u32 {
     var builder = Builder.init(arena.child_allocator, handle, encoding);
 
+    // as line comments are not nodes, we parse the text then generate the tokens for them
+    try findComments(&builder, handle.tree.source, encoding);
+
     // reverse the ast from the root declarations
     var gap_highlighter = GapHighlighter.init(&builder, 0);
+
     var buf: [2]ast.Node.Index = undefined;
     for (analysis.declMembers(handle.tree, .root, 0, &buf)) |child| {
         try gap_highlighter.next(child);
@@ -1020,7 +1058,57 @@ pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentS
     }
 
     try gap_highlighter.end(@truncate(u32, handle.tree.tokens.len) - 1);
-    // pass root node, which always has index '0'
+
-    // try writeNodeTokens(&builder, arena, store, 0);
     return builder.toOwnedSlice();
 }
+
+/// As the AST does not contain nodes for comments
+/// this will parse through the entire file to search for comments
+/// and generate semantic tokens for them
+fn findComments(builder: *Builder, source: []const u8, encoding: offsets.Encoding) !void {
+    var state: enum { none, comment, doc_comment } = .none;
+
+    var prev: u8 = 0;
+    var start: usize = 0;
+    for (source) |c, i| {
+        if (state == .comment and c == '/') {
+            state = .none;
+            continue;
+        }
+
+        if (state == .none and c == '/' and prev == '/') {
+            state = .comment;
+            start = i - 1;
+        }
+
+        if (c == '\n') {
+            if (state == .comment) {
+                state = .none;
+
+                const len = if (encoding == .utf8)
+                    i - start
+                else blk: {
+                    var index: usize = start;
+                    var utf16_len: usize = 0;
+                    while (index < i) {
+                        const n = std.unicode.utf8ByteSequenceLength(source[index]) catch unreachable;
+                        const codepoint = std.unicode.utf8Decode(source[index .. index + n]) catch unreachable;
+                        if (codepoint < 0x10000) {
+                            utf16_len += 1;
+                        } else {
+                            utf16_len += 2;
+                        }
+                        index += n;
+                    }
+                    break :blk utf16_len;
+                };
+
+                try builder.comments.append(.{
+                    .length = @truncate(u32, len),
+                    .start = @truncate(u32, start),
+                });
+            }
+        }
+        prev = c;
+    }
+}