From 9224bbd4dc77e9bc415270d581f92725a966a42e Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Thu, 4 Mar 2021 22:30:25 +0100 Subject: [PATCH] Implement all of the semantic tokens --- src/analysis.zig | 41 +- src/semantic_tokens.zig | 876 ++++++++++++++++++++++++---------------- 2 files changed, 544 insertions(+), 373 deletions(-) diff --git a/src/analysis.zig b/src/analysis.zig index 519d1fa..c0814f2 100644 --- a/src/analysis.zig +++ b/src/analysis.zig @@ -30,13 +30,13 @@ pub fn getDocCommentTokenIndex(tree: ast.Tree, node: ast.Node.Index) ?ast.TokenI idx -= 2; // skip '.' token }, else => { - // if (isContainer(tags[node])) { - // idx -= 1; // go to '=' - // idx -= 1; // mutability - // idx -= 1; // possible 'pub' - // if (tokens[idx] == .keyword_pub and idx > 0) - // idx -= 1; - // } + if (isContainer(tags[node])) { + idx -= 1; // go to '=' + idx -= 1; // mutability + idx -= 1; // possible 'pub' + if (tokens[idx] == .keyword_pub and idx > 0) + idx -= 1; + } else log.debug("Doc comment check for tag: {s}", .{tags[node]}); }, } @@ -187,7 +187,7 @@ pub fn isTypeFunction(tree: ast.Tree, func: ast.full.FnProto) bool { } pub fn isGenericFunction(tree: ast.Tree, func: ast.full.FnProto) bool { - var it = func.iterate(); + var it = func.iterate(tree); while (it.next()) |param| { if (param.anytype_ellipsis3 != null or param.comptime_noalias != null) { return true; @@ -983,7 +983,7 @@ pub const TypeWithHandle = struct { } } - fn isContainer(self: TypeWithHandle, container_kind_tok: std.zig.Token.Tag, tree: ast.Tree) bool { + fn isContainerKind(self: TypeWithHandle, container_kind_tok: std.zig.Token.Tag, tree: ast.Tree) bool { const main_tokens = tree.nodes.items(.main_token); const tags = tree.tokens.items(.tag); switch (self.type.data) { @@ -993,30 +993,33 @@ pub const TypeWithHandle = struct { } pub fn isStructType(self: TypeWithHandle, tree: ast.Tree) bool { - return self.isContainer(.keyword_struct, tree) or self.isRoot(); + return self.isContainerKind(.keyword_struct, tree) or self.isRoot(); } pub fn isNamespace(self: TypeWithHandle, tree: ast.Tree) bool { - if (!self.isStructType()) return false; - var idx: usize = 0; - // @TODO: FIX ME - while (self.type.data.other.iterate(idx)) |child| : (idx += 1) { - if (child.tag == .ContainerField) - return false; + if (!self.isStructType(tree)) return false; + + const node = self.type.data.other; + const tags = tree.nodes.items(.tag); + if (isContainer(tags[node])) { + var buf: [2]ast.Node.Index = undefined; + for (declMembers(tree, tags[node], node, &buf)) |child| { + if (tags[child].isContainerField()) return false; + } } return true; } pub fn isEnumType(self: TypeWithHandle, tree: ast.Tree) bool { - return self.isContainer(.keyword_enum, tree); + return self.isContainerKind(.keyword_enum, tree); } pub fn isUnionType(self: TypeWithHandle, tree: ast.Tree) bool { - return self.isContainer(.keyword_union, tree); + return self.isContainerKind(.keyword_union, tree); } pub fn isOpaqueType(self: TypeWithHandle, tree: ast.Tree) bool { - return self.isContainer(.keyword_opaque, tree); + return self.isContainerKind(.keyword_opaque, tree); } pub fn isTypeFunc(self: TypeWithHandle, tree: ast.Tree) bool { diff --git a/src/semantic_tokens.zig b/src/semantic_tokens.zig index e01aae6..344e223 100644 --- a/src/semantic_tokens.zig +++ b/src/semantic_tokens.zig @@ -65,11 +65,11 @@ const Builder = struct { fn add(self: *Builder, token: ast.TokenIndex, token_type: TokenType, token_modifiers: TokenModifiers) !void { const start_idx = if (self.current_token) |current_token| - self.handle.tree.tokenLocation[current_token].line_start + self.handle.tree.tokenLocation(0, current_token).line_start else 0; - if (start_idx > self.handle.tree.tokenLocation[token].line_start) + if (start_idx > self.handle.tree.tokenLocation(0, token).line_start) return; const delta_loc = offsets.tokenRelativeLocation(self.handle.tree, start_idx, token, self.encoding) catch return; @@ -107,7 +107,7 @@ fn writeTokenMod( } } -fn writeDocComments(builder: *Builder, tree: *ast.Tree, doc: *ast.TokenIndex) !void { +fn writeDocComments(builder: *Builder, tree: ast.Tree, doc: ast.TokenIndex) !void { const token_tags = tree.tokens.items(.tag); var tok_idx = doc; while (token_tags[tok_idx] == .doc_comment or @@ -120,10 +120,11 @@ fn writeDocComments(builder: *Builder, tree: *ast.Tree, doc: *ast.TokenIndex) !v } } -fn fieldTokenType(container_decl: *ast.full.ContainerDecl, handle: *DocumentStore.Handle) ?TokenType { - if (container_decl.ast.main_token > handle.tree.tokens.len) return null; - return @as(?TokenType, switch (handle.tree.tokens.items(.tag)[container_decl.ast.main_token]) { - .keyword_Struct => .field, +fn fieldTokenType(container_decl: ast.Node.Index, handle: *DocumentStore.Handle) ?TokenType { + const main_token = handle.tree.nodes.items(.main_token)[container_decl]; + if (main_token > handle.tree.tokens.len) return null; + return @as(?TokenType, switch (handle.tree.tokens.items(.tag)[main_token]) { + .keyword_struct => .field, .keyword_union, .keyword_enum => .enumMember, else => null, }); @@ -158,7 +159,7 @@ const GapHighlighter = struct { } else if (@enumToInt(tok_id) >= @enumToInt(std.zig.Token.Tag.bang) and @enumToInt(tok_id) <= @enumToInt(std.zig.Token.Tag.tilde) and tok_id != .period and tok_id != .comma and tok_id != .r_paren and - tok_id != .l_paren and tok_id != .r_bracce and tok_id != .l_brace and + tok_id != .l_paren and tok_id != .r_brace and tok_id != .l_brace and tok_id != .semicolon and tok_id != .colon) { try writeToken(self.builder, tok, .operator); @@ -173,16 +174,17 @@ const GapHighlighter = struct { return .{ .builder = builder, .current_idx = start }; } - fn next(self: *GapHighlighter, node: *ast.Node) !void { - if (self.current_idx > 0 and self.builder.handle.tree.token_ids[self.current_idx - 1] == .container_doc_comment) { + fn next(self: *GapHighlighter, node: ast.Node.Index) !void { + const tree = self.builder.handle.tree; + if (self.current_idx > 0 and tree.tokens.items(.tag)[self.current_idx - 1] == .container_doc_comment) { try self.handleTok(self.current_idx - 1); } var i = self.current_idx; - while (i < node.firstToken()) : (i += 1) { + while (i < tree.firstToken(node)) : (i += 1) { try self.handleTok(i); } - self.current_idx = node.lastToken() + 1; + self.current_idx = tree.lastToken(node) + 1; } fn end(self: *GapHighlighter, last: ast.TokenIndex) !void { @@ -194,25 +196,26 @@ const GapHighlighter = struct { }; fn colorIdentifierBasedOnType(builder: *Builder, type_node: analysis.TypeWithHandle, target_tok: ast.TokenIndex, tok_mod: TokenModifiers) !void { + const tree = builder.handle.tree; if (type_node.type.is_type_val) { var new_tok_mod = tok_mod; - if (type_node.isNamespace()) + if (type_node.isNamespace(tree)) new_tok_mod.set("namespace") - else if (type_node.isStructType()) + else if (type_node.isStructType(tree)) new_tok_mod.set("struct") - else if (type_node.isEnumType()) + else if (type_node.isEnumType(tree)) new_tok_mod.set("enum") - else if (type_node.isUnionType()) + else if (type_node.isUnionType(tree)) new_tok_mod.set("union") - else if (type_node.isOpaqueType()) + else if (type_node.isOpaqueType(tree)) new_tok_mod.set("opaque"); try writeTokenMod(builder, target_tok, .type, new_tok_mod); - } else if (type_node.isTypeFunc()) { + } else if (type_node.isTypeFunc(tree)) { try writeTokenMod(builder, target_tok, .type, tok_mod); - } else if (type_node.isFunc()) { + } else if (type_node.isFunc(tree)) { var new_tok_mod = tok_mod; - if (type_node.isGenericFunc()) { + if (type_node.isGenericFunc(tree)) { new_tok_mod.set("generic"); } try writeTokenMod(builder, target_tok, .function, new_tok_mod); @@ -225,29 +228,35 @@ fn writeContainerField( builder: *Builder, arena: *std.heap.ArenaAllocator, store: *DocumentStore, - container_field: *ast.Node.ContainerField, + node: ast.Node.Index, field_token_type: ?TokenType, child_frame: anytype, ) !void { - if (container_field.doc_comments) |docs| try writeDocComments(builder, builder.handle.tree, docs); + const container_field = analysis.containerField(builder.handle.tree, node).?; + if (analysis.getDocCommentTokenIndex(builder.handle.tree, node)) |docs| + try writeDocComments(builder, builder.handle.tree, docs); + try writeToken(builder, container_field.comptime_token, .keyword); - if (field_token_type) |tok_type| try writeToken(builder, container_field.name_token, tok_type); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.type_expr }); - if (container_field.align_expr) |n| { - try writeToken(builder, n.firstToken() - 2, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, n }); + if (field_token_type) |tok_type| try writeToken(builder, container_field.ast.name_token, tok_type); + + if (container_field.ast.type_expr != 0) { + if (container_field.ast.align_expr != 0) { + try writeToken(builder, builder.handle.tree.firstToken(container_field.ast.align_expr) - 2, .keyword); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.ast.align_expr }); + } + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.ast.type_expr }); } - if (container_field.value_expr) |value_expr| block: { - const eq_tok: ast.TokenIndex = if (container_field.type_expr) |type_expr| - type_expr.lastToken() + 1 - else if (container_field.align_expr) |align_expr| - align_expr.lastToken() + 1 + if (container_field.ast.value_expr != 0) block: { + const eq_tok: ast.TokenIndex = if (container_field.ast.type_expr != 0) + builder.handle.tree.lastToken(container_field.ast.type_expr) + 1 + else if (container_field.ast.align_expr != 0) + builder.handle.tree.lastToken(container_field.ast.align_expr) + 1 else break :block; // Check this, I believe it is correct. try writeToken(builder, eq_tok, .operator); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, value_expr }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.ast.value_expr }); } } @@ -257,18 +266,19 @@ fn writeNodeTokens( arena: *std.heap.ArenaAllocator, store: *DocumentStore, maybe_node: ?ast.Node.Index, - tree: ast.Tree, ) error{OutOfMemory}!void { if (maybe_node == null) return; - const node = maybe_node.?; + const handle = builder.handle; + const tree = handle.tree; const node_tags = tree.nodes.items(.tag); const token_tags = tree.tokens.items(.tag); const datas = tree.nodes.items(.data); const main_tokens = tree.nodes.items(.main_token); + + const node = maybe_node.?; const tag = node_tags[node]; const main_token = main_tokens[node]; - const handle = builder.handle; const FrameSize = @sizeOf(@Frame(writeNodeTokens)); var child_frame = try arena.child_allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize); @@ -281,15 +291,19 @@ fn writeNodeTokens( for (analysis.declMembers(tree, .root, 0, &buf)) |child| { try gap_highlighter.next(child); if (node_tags[child].isContainerField()) { - try writeContainerField(builder, arena, store, analysis.containerField(tree, child), .field, child_frame); + try writeContainerField(builder, arena, store, child, .field, child_frame); } else { try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, child }); } } - try gap_highlighter.end(handle.tree.tokens.len - 1); + try gap_highlighter.end(@truncate(u32, tree.tokens.len) - 1); }, - .block, .block_semicolon, .block_two, .block_two_semicolon => { - const first_tok = if (token_tags[main_token - 1] == .colon and token_tags[main_token - 2] == identifier) block: { + .block, + .block_semicolon, + .block_two, + .block_two_semicolon, + => { + const first_tok = if (token_tags[main_token - 1] == .colon and token_tags[main_token - 2] == .identifier) block: { try writeToken(builder, main_token - 2, .label); break :block main_token + 1; } else 0; @@ -313,7 +327,7 @@ fn writeNodeTokens( for (statements) |child| { try gap_highlighter.next(child); if (node_tags[child].isContainerField()) { - try writeContainerField(builder, arena, store, analysis.containerField(tree, child), .field, child_frame); + try writeContainerField(builder, arena, store, child, .field, child_frame); } else { try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, child }); } @@ -321,7 +335,11 @@ fn writeNodeTokens( try gap_highlighter.end(tree.lastToken(node)); }, - .global_var_decl, .local_var_decl, .simple_var_decl, .aligned_var_decl => { + .global_var_decl, + .local_var_decl, + .simple_var_decl, + .aligned_var_decl, + => { const var_decl = analysis.varDecl(tree, node).?; if (analysis.getDocCommentTokenIndex(tree, node)) |comment_idx| try writeDocComments(builder, handle.tree, comment_idx); @@ -331,14 +349,20 @@ fn writeNodeTokens( try writeToken(builder, var_decl.threadlocal_token, .keyword); try writeToken(builder, var_decl.comptime_token, .keyword); try writeToken(builder, var_decl.ast.mut_token, .keyword); + if (try analysis.resolveTypeOfNode(store, arena, .{ .node = node, .handle = handle })) |decl_type| { try colorIdentifierBasedOnType(builder, decl_type, var_decl.ast.mut_token + 1, .{ .declaration = true }); } else { try writeTokenMod(builder, var_decl.ast.mut_token + 1, .variable, .{ .declaration = true }); } - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.type_node }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.align_node }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.section_node }); + + if (var_decl.ast.type_node != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.type_node }); + if (var_decl.ast.align_node != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.align_node }); + if (var_decl.ast.section_node != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.section_node }); + try writeToken(builder, var_decl.ast.mut_token + 2, .operator); try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.init_node }); }, @@ -350,9 +374,6 @@ fn writeNodeTokens( try writeToken(builder, main_token, .keyword); try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); }, - .error_set_decl => { - // @TODO: Semantic highlighting for error set decl - }, .container_decl, .container_decl_trailing, .container_decl_two, @@ -370,7 +391,7 @@ fn writeNodeTokens( try writeToken(builder, decl.layout_token, .keyword); try writeToken(builder, decl.ast.main_token, .keyword); - if (decl.enum_token) |enum_token| { + if (decl.ast.enum_token) |enum_token| { if (decl.ast.arg != 0) try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, decl.ast.arg }) else @@ -378,11 +399,11 @@ fn writeNodeTokens( } else if (decl.ast.arg != 0) try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, decl.ast.arg }); var gap_highlighter = GapHighlighter.init(builder, main_token + 1); - const field_token_type = fieldTokenType(decl, handle); + const field_token_type = fieldTokenType(node, handle); for (decl.ast.members) |child| { try gap_highlighter.next(child); - if (node_tags[node].isContainerField()) { - try writeContainerField(builder, arena, store, analysis.containerField(tree, node), field_token_type, child_frame); + if (node_tags[child].isContainerField()) { + try writeContainerField(builder, arena, store, child, field_token_type, child_frame); } else { try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, child }); } @@ -398,7 +419,13 @@ fn writeNodeTokens( return try writeToken(builder, main_token, .type); } - if (try analysis.lookupSymbolGlobal(store, arena, handle, handle.tree.getNodeSource(node), handle.tree.items(.start)[main_token])) |child| { + if (try analysis.lookupSymbolGlobal( + store, + arena, + handle, + handle.tree.getNodeSource(node), + handle.tree.tokens.items(.start)[main_token], + )) |child| { if (child.decl.* == .param_decl) { return try writeToken(builder, main_token, .parameter); } @@ -410,10 +437,15 @@ fn writeNodeTokens( } } }, - .fn_proto, .fn_proto_one, .fn_proto_simple, .fn_proto_multiple, .fn_decl => { + .fn_proto, + .fn_proto_one, + .fn_proto_simple, + .fn_proto_multi, + .fn_decl, + => { var buf: [1]ast.Node.Index = undefined; const fn_proto: ast.full.FnProto = analysis.fnProto(tree, node, &buf).?; - if (analysis.getDocCommentTokenIndex(tree, node)) |cocs| + if (analysis.getDocCommentTokenIndex(tree, node)) |docs| try writeDocComments(builder, handle.tree, docs); try writeToken(builder, fn_proto.visib_token, .keyword); @@ -440,16 +472,16 @@ fn writeNodeTokens( try writeToken(builder, param_decl.comptime_noalias, .keyword); try writeTokenMod(builder, param_decl.name_token, .parameter, .{ .declaration = true }); if (param_decl.anytype_ellipsis3) |any_token| { - try writeToken(builder, var_node.firstToken(), .type); + try writeToken(builder, any_token, .type); } else if (param_decl.type_expr != 0) try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param_decl.type_expr }); } if (fn_proto.ast.align_expr != 0) - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.align_expr != 0 }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.align_expr }); if (fn_proto.ast.section_expr != 0) - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.section_expr != 0 }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.section_expr }); if (fn_proto.ast.callconv_expr != 0) - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.callconv_expr != 0 }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.callconv_expr }); if (fn_proto.ast.return_type != 0) try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.return_type }); @@ -468,26 +500,32 @@ fn writeNodeTokens( try writeToken(builder, main_token, .keyword); try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].rhs }); }, - .@"comptime", @"nosuspend" => { + .@"comptime", + .@"nosuspend", + => { if (analysis.getDocCommentTokenIndex(tree, node)) |doc| try writeDocComments(builder, handle.tree, doc); try writeToken(builder, main_token, .keyword); try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); }, - .@"switch", .switch_comma => { + .@"switch", + .switch_comma, + => { try writeToken(builder, main_token, .keyword); try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); const extra = tree.extraData(datas[node].rhs, ast.Node.SubRange); const cases = tree.extra_data[extra.start..extra.end]; - var gap_highlighter = GapHighlighter.init(builder, switch_node.expr.lastToken() + 3); + var gap_highlighter = GapHighlighter.init(builder, tree.lastToken(datas[node].lhs) + 1); for (cases) |case_node| { try gap_highlighter.next(case_node); try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, case_node }); } - try gap_highlighter.end(node.lastToken()); + try gap_highlighter.end(tree.lastToken(node)); }, - .switch_case_one, .switch_case => { + .switch_case_one, + .switch_case, + => { const switch_case = if (tag == .switch_case) tree.switchCase(node) else tree.switchCaseOne(node); for (switch_case.ast.values) |item_node| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, item_node }); // check it it's 'else' @@ -499,342 +537,472 @@ fn writeNodeTokens( } try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, switch_case.ast.target_expr }); }, - .@"while", .while_simple, .while_cont, .for_simple, .@"for" => { - const while_node: ast.full.While = switch (node) { - .@"while" => tree.whileFull(node_idx), - .while_simple => tree.whileSimple(node_idx), - .while_cont => tree.whileCont(node_idx), - .@"for" => tree.forFull(node_idx), - .for_simple => tree.forSimple(node_idx), + .@"while", + .while_simple, + .while_cont, + .for_simple, + .@"for", + => { + const while_node: ast.full.While = switch (tag) { + .@"while" => tree.whileFull(node), + .while_simple => tree.whileSimple(node), + .while_cont => tree.whileCont(node), + .@"for" => tree.forFull(node), + .for_simple => tree.forSimple(node), else => unreachable, }; try writeToken(builder, while_node.label_token, .label); try writeToken(builder, while_node.inline_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.condition }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.cond_expr }); try writeToken(builder, while_node.payload_token, .variable); if (while_node.ast.cont_expr != 0) - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.cont_expr}); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.body }); - if (while_node.@"else") |else_node| - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, &else_node.base }); - }, - .For => { - const for_node = node.cast(ast.Node.For).?; - try writeToken(builder, for_node.label, .label); - try writeToken(builder, for_node.inline_token, .keyword); - try writeToken(builder, for_node.for_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, for_node.array_expr }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, for_node.payload }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, for_node.body }); - if (for_node.@"else") |else_node| - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, &else_node.base }); - }, - .If => { - const if_node = node.cast(ast.Node.If).?; - try writeToken(builder, if_node.if_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.condition }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.payload }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.body }); - if (if_node.@"else") |else_node| - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, &else_node.base }); - }, - .ArrayInitializer => { - const array_initializer = node.cast(ast.Node.ArrayInitializer).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_initializer.lhs }); - for (array_initializer.listConst()) |elem| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, elem }); - }, - .ArrayInitializerDot => { - const array_initializer = node.cast(ast.Node.ArrayInitializerDot).?; - for (array_initializer.listConst()) |elem| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, elem }); - }, - .StructInitializer => { - const struct_initializer = node.cast(ast.Node.StructInitializer).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, struct_initializer.lhs }); - const field_token_type = if (try analysis.resolveTypeOfNode(store, arena, .{ .node = struct_initializer.lhs, .handle = handle })) |struct_type| switch (struct_type.type.data) { - .other => |type_node| if (type_node.cast(ast.Node.ContainerDecl)) |container_decl| - fieldTokenType(container_decl, handle) - else - null, - else => null, - } else null; + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.cont_expr }); - var gap_highlighter = GapHighlighter.init(builder, struct_initializer.lhs.lastToken() + 1); - for (struct_initializer.listConst()) |field_init_node| { - try gap_highlighter.next(field_init_node); - std.debug.assert(field_init_node.tag == .FieldInitializer); - const field_init = field_init_node.cast(ast.Node.FieldInitializer).?; + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.then_expr }); + + try writeToken(builder, while_node.error_token, .variable); + + if (while_node.ast.else_expr != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.else_expr }); + }, + .@"if", + .if_simple, + => { + const if_node: ast.full.If = if (tag == .@"if") tree.ifFull(node) else tree.ifSimple(node); + + try writeToken(builder, if_node.ast.if_token, .keyword); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.ast.cond_expr }); + + try writeToken(builder, if_node.payload_token, .variable); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.ast.then_expr }); + + try writeToken(builder, if_node.error_token, .variable); + if (if_node.ast.else_expr != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.ast.else_expr }); + }, + .array_init, + .array_init_comma, + .array_init_one, + .array_init_one_comma, + .array_init_dot, + .array_init_dot_comma, + .array_init_dot_two, + .array_init_dot_two_comma, + => { + var buf: [2]ast.Node.Index = undefined; + const array_init: ast.full.ArrayInit = switch (tag) { + .array_init, .array_init_comma => tree.arrayInit(node), + .array_init_one, .array_init_one_comma => tree.arrayInitOne(buf[0..1], node), + .array_init_dot, .array_init_dot_comma => tree.arrayInitDot(node), + .array_init_dot_two, .array_init_dot_two_comma => tree.arrayInitDotTwo(&buf, node), + else => unreachable, + }; + + if (array_init.ast.type_expr != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_init.ast.type_expr }); + for (array_init.ast.elements) |elem| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, elem }); + }, + .struct_init, + .struct_init_comma, + .struct_init_dot, + .struct_init_dot_comma, + .struct_init_one, + .struct_init_one_comma, + .struct_init_dot_two, + .struct_init_dot_two_comma, + => { + var buf: [2]ast.Node.Index = undefined; + const struct_init: ast.full.StructInit = switch (tag) { + .struct_init, .struct_init_comma => tree.structInit(node), + .struct_init_dot, .struct_init_dot_comma => tree.structInitDot(node), + .struct_init_one, .struct_init_one_comma => tree.structInitOne(buf[0..1], node), + .struct_init_dot_two, .struct_init_dot_two_comma => tree.structInitDotTwo(&buf, node), + else => unreachable, + }; + + var field_token_type: ?TokenType = null; + + if (struct_init.ast.type_expr != 0) { + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, struct_init.ast.type_expr }); + + field_token_type = if (try analysis.resolveTypeOfNode(store, arena, .{ + .node = struct_init.ast.type_expr, + .handle = handle, + })) |struct_type| switch (struct_type.type.data) { + .other => |type_node| if (analysis.isContainer(struct_type.handle.tree.nodes.items(.tag)[type_node])) + fieldTokenType(type_node, handle) + else + null, + else => null, + } else null; + } + + var gap_highlighter = GapHighlighter.init(builder, struct_init.ast.lbrace); + for (struct_init.ast.fields) |field_init| { + try gap_highlighter.next(field_init); + + const init_token = tree.firstToken(field_init); if (field_token_type) |tok_type| { - try writeToken(builder, field_init.period_token, tok_type); - try writeToken(builder, field_init.name_token, tok_type); + try writeToken(builder, init_token - 3, tok_type); + try writeToken(builder, init_token - 2, tok_type); } - try writeToken(builder, field_init.name_token + 1, .operator); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, field_init.expr }); + try writeToken(builder, init_token - 1, .operator); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, field_init }); } - try gap_highlighter.end(struct_initializer.rtoken); + try gap_highlighter.end(tree.lastToken(node)); }, - .StructInitializerDot => { - const struct_initializer = node.castTag(.StructInitializerDot).?; + .call, + .call_comma, + .async_call, + .async_call_comma, + .call_one, + .call_one_comma, + .async_call_one, + .async_call_one_comma, + => { + var params: [1]ast.Node.Index = undefined; + const call: ast.full.Call = switch (tag) { + .call, .call_comma, .async_call, .async_call_comma => tree.callFull(node), + .call_one, .call_one_comma, .async_call_one, .async_call_one_comma => tree.callOne(¶ms, node), + else => unreachable, + }; - var gap_highlighter = GapHighlighter.init(builder, struct_initializer.dot + 1); - for (struct_initializer.listConst()) |field_init_node| { - try gap_highlighter.next(field_init_node); - std.debug.assert(field_init_node.tag == .FieldInitializer); - const field_init = field_init_node.castTag(.FieldInitializer).?; - try writeToken(builder, field_init.period_token, .field); - try writeToken(builder, field_init.name_token, .field); - try writeToken(builder, field_init.name_token + 1, .operator); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, field_init.expr }); - } - try gap_highlighter.end(struct_initializer.rtoken); - }, - .Call => { - const call = node.cast(ast.Node.Call).?; try writeToken(builder, call.async_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, call.lhs }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, call.ast.fn_expr }); + if (builder.current_token) |curr_tok| { - if (curr_tok != call.lhs.lastToken() and handle.tree.token_ids[call.lhs.lastToken()] == .Identifier) { - try writeToken(builder, call.lhs.lastToken(), .function); + if (curr_tok != tree.lastToken(call.ast.fn_expr) and token_tags[tree.lastToken(call.ast.fn_expr)] == .identifier) { + try writeToken(builder, tree.lastToken(call.ast.fn_expr), .function); } } - for (call.paramsConst()) |param| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param }); + for (call.ast.params) |param| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param }); }, - .Slice => { - const slice = node.castTag(.Slice).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.lhs }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.start }); - try writeToken(builder, slice.start.lastToken() + 1, .operator); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.end }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.sentinel }); + .slice, + .slice_open, + .slice_sentinel, + => { + const slice: ast.full.Slice = switch (tag) { + .slice => tree.slice(node), + .slice_open => tree.sliceOpen(node), + .slice_sentinel => tree.sliceSentinel(node), + else => unreachable, + }; + + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.sliced }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.start }); + try writeToken(builder, tree.lastToken(slice.ast.start) + 1, .operator); + + if (slice.ast.end != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.end }); + if (slice.ast.sentinel != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.sentinel }); }, - .ArrayAccess => { - const arr_acc = node.castTag(.ArrayAccess).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, arr_acc.lhs }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, arr_acc.index_expr }); + .array_access => { + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].rhs }); }, - .Deref, .UnwrapOptional => { - const suffix = node.cast(ast.Node.SimpleSuffixOp).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, suffix.lhs }); - try writeToken(builder, suffix.rtoken, .operator); + .deref, + .unwrap_optional, + => { + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); + try writeToken(builder, main_token, .operator); }, - .GroupedExpression => { - const grouped_expr = node.cast(ast.Node.GroupedExpression).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, grouped_expr.expr }); + .grouped_expression => { + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); }, - .Return, .Break, .Continue => { - const cfe = node.cast(ast.Node.ControlFlowExpression).?; - try writeToken(builder, cfe.ltoken, .keyword); - switch (node.tag) { - .Break => if (cfe.getLabel()) |n| try writeToken(builder, n, .label), - .Continue => if (cfe.getLabel()) |n| try writeToken(builder, n, .label), - else => {}, - } - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, cfe.getRHS() }); + .@"return", + .@"break", + .@"continue", + => { + try writeToken(builder, main_token, .keyword); + if (datas[node].lhs != 0) + try writeToken(builder, datas[node].lhs, .label); + if (datas[node].rhs != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].rhs }); }, - .Suspend => { - const suspend_node = node.cast(ast.Node.Suspend).?; - try writeToken(builder, suspend_node.suspend_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, suspend_node.body }); + .@"suspend" => { + try writeToken(builder, main_token, .keyword); + if (datas[node].lhs != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); }, - .IntegerLiteral => { - try writeToken(builder, node.firstToken(), .number); + .integer_literal, + .float_literal, + => { + try writeToken(builder, main_token, .number); }, - .EnumLiteral => { - const enum_literal = node.cast(ast.Node.EnumLiteral).?; - try writeToken(builder, enum_literal.dot, .enumMember); - try writeToken(builder, enum_literal.name, .enumMember); + .enum_literal => { + try writeToken(builder, main_token - 1, .enumMember); + try writeToken(builder, main_token, .enumMember); }, - .FloatLiteral => { - try writeToken(builder, node.firstToken(), .number); - }, - .BuiltinCall => { - const builtin_call = node.cast(ast.Node.BuiltinCall).?; - try writeToken(builder, builtin_call.builtin_token, .builtin); - for (builtin_call.paramsConst()) |param| + .builtin_call, + .builtin_call_comma, + .builtin_call_two, + .builtin_call_two_comma, + => { + const data = datas[node]; + const params = switch (tag) { + .builtin_call, .builtin_call_comma => tree.extra_data[data.lhs..data.rhs], + .builtin_call_two, .builtin_call_two_comma => if (data.lhs == 0) + &[_]ast.Node.Index{} + else if (data.rhs == 0) + &[_]ast.Node.Index{data.lhs} + else + &[_]ast.Node.Index{ data.lhs, data.rhs }, + else => unreachable, + }; + + try writeToken(builder, main_token, .builtin); + for (params) |param| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param }); }, - .StringLiteral, .CharLiteral => { - try writeToken(builder, node.firstToken(), .string); + .string_literal, + .char_literal, + => { + try writeToken(builder, main_token, .string); }, - .MultilineStringLiteral => { - const multi_line = node.cast(ast.Node.MultilineStringLiteral).?; - for (multi_line.linesConst()) |line| try writeToken(builder, line, .string); + .multiline_string_literal => { + var cur_tok = main_token; + const last_tok = datas[node].rhs; + + while (cur_tok <= last_tok) : (cur_tok += 1) try writeToken(builder, cur_tok, .string); }, - .BoolLiteral, .NullLiteral, .UndefinedLiteral, .Unreachable => { - try writeToken(builder, node.firstToken(), .keywordLiteral); + .true_literal, + .false_literal, + .null_literal, + .undefined_literal, + .unreachable_literal, + => { + try writeToken(builder, main_token, .keywordLiteral); }, - .ErrorType => { - try writeToken(builder, node.firstToken(), .keyword); + .error_set_decl => { + try writeToken(builder, main_token, .keyword); }, - .Asm => { - const asm_expr = node.cast(ast.Node.Asm).?; - try writeToken(builder, asm_expr.asm_token, .keyword); - try writeToken(builder, asm_expr.volatile_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, asm_expr.template }); + .@"asm", + .asm_output, + .asm_input, + .asm_simple, + => { + const asm_node: ast.full.Asm = switch (tag) { + .@"asm" => tree.asmFull(node), + .asm_simple => tree.asmSimple(node), + else => return, // TODO Inputs, outputs + }; + + try writeToken(builder, main_token, .keyword); + try writeToken(builder, asm_node.volatile_token, .keyword); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, asm_node.ast.template }); // TODO Inputs, outputs. }, - .AnyType => { - try writeToken(builder, node.firstToken(), .type); + .@"anytype" => { + try writeToken(builder, main_token, .type); }, - .TestDecl => { - const test_decl = node.cast(ast.Node.TestDecl).?; - if (test_decl.doc_comments) |doc| try writeDocComments(builder, handle.tree, doc); - try writeToken(builder, test_decl.test_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, test_decl.name }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, test_decl.body_node }); + .test_decl => { + if (analysis.getDocCommentTokenIndex(handle.tree, node)) |doc| + try writeDocComments(builder, handle.tree, doc); + + try writeToken(builder, main_token, .keyword); + if (token_tags[main_token + 1] == .string_literal) + try writeToken(builder, main_token + 1, .string); + + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].rhs }); }, - .Catch => { - const catch_expr = node.cast(ast.Node.Catch).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, catch_expr.lhs }); - try writeToken(builder, catch_expr.op_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, catch_expr.rhs }); + .@"catch" => { + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); + try writeToken(builder, main_token, .keyword); + if (token_tags[main_token + 1] == .pipe) + try writeToken(builder, main_token + 1, .variable); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].rhs }); }, - .Add, .AddWrap, .ArrayCat, .ArrayMult, .Assign, .AssignBitAnd, .AssignBitOr, .AssignBitShiftLeft, .AssignBitShiftRight, .AssignBitXor, .AssignDiv, .AssignSub, .AssignSubWrap, .AssignMod, .AssignAdd, .AssignAddWrap, .AssignMul, .AssignMulWrap, .BangEqual, .BitAnd, .BitOr, .BitShiftLeft, .BitShiftRight, .BitXor, .BoolAnd, .BoolOr, .Div, .EqualEqual, .ErrorUnion, .GreaterOrEqual, .GreaterThan, .LessOrEqual, .LessThan, .MergeErrorSets, .Mod, .Mul, .MulWrap, .Period, .Range, .Sub, .SubWrap, .OrElse => { - const infix_op = node.cast(ast.Node.SimpleInfixOp).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, infix_op.lhs }); - if (node.tag != .Period) { - const token_type: TokenType = switch (node.tag) { - .BoolAnd, .BoolOr, .OrElse => .keyword, - else => .operator, - }; - - try writeToken(builder, infix_op.op_token, token_type); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, infix_op.rhs }); - } - switch (node.tag) { - .Period => { - const rhs_str = handle.tree.tokenSlice(infix_op.rhs.firstToken()); - - // TODO This is basically exactly the same as what is done in analysis.resolveTypeOfNode, with the added - // writeToken code. - // Maybe we can hook into it insead? Also applies to Identifier and VarDecl - var bound_type_params = analysis.BoundTypeParams.init(&arena.allocator); - const lhs_type = try analysis.resolveFieldAccessLhsType( - store, - arena, - (try analysis.resolveTypeOfNodeInternal(store, arena, .{ - .node = infix_op.lhs, - .handle = handle, - }, &bound_type_params)) orelse return, - &bound_type_params, - ); - const left_type_node = switch (lhs_type.type.data) { - .other => |n| n, - else => return, - }; - if (try analysis.lookupSymbolContainer(store, arena, .{ .node = left_type_node, .handle = lhs_type.handle }, rhs_str, !lhs_type.type.is_type_val)) |decl_type| { - switch (decl_type.decl.*) { - .ast_node => |decl_node| { - if (decl_node.tag == .ContainerField) { - const tok_type: ?TokenType = if (left_type_node.cast(ast.Node.ContainerDecl)) |container_decl| - fieldTokenType(container_decl, lhs_type.handle) - else if (left_type_node.tag == .Root) - TokenType.field - else - null; - - if (tok_type) |tt| try writeToken(builder, infix_op.rhs.firstToken(), tt); - return; - } else if (decl_node.tag == .ErrorTag) { - try writeToken(builder, infix_op.rhs.firstToken(), .errorTag); - } - }, - else => {}, - } - - if (try decl_type.resolveType(store, arena, &bound_type_params)) |resolved_type| { - try colorIdentifierBasedOnType(builder, resolved_type, infix_op.rhs.firstToken(), .{}); - } - } - }, - else => {}, - } - }, - .SliceType => { - const slice_type = node.castTag(.SliceType).?; - const ptr_info = slice_type.ptr_info; - if (ptr_info.align_info) |align_info| { - try writeToken(builder, slice_type.op_token + 2, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, align_info.node }); - } - try writeToken(builder, ptr_info.const_token, .keyword); - try writeToken(builder, ptr_info.volatile_token, .keyword); - try writeToken(builder, ptr_info.allowzero_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice_type.rhs }); - }, - .PtrType => { - const pointer_type = node.castTag(.PtrType).?; - const tok_ids = builder.handle.tree.token_ids; - - const ptr_info = switch (tok_ids[pointer_type.op_token]) { - .AsteriskAsterisk => pointer_type.rhs.castTag(.PtrType).?.ptr_info, - else => pointer_type.ptr_info, - }; - const rhs = switch (tok_ids[pointer_type.op_token]) { - .AsteriskAsterisk => pointer_type.rhs.castTag(.PtrType).?.rhs, - else => pointer_type.rhs, - }; - - const off = switch (tok_ids[pointer_type.op_token]) { - .Asterisk, .AsteriskAsterisk => blk: { - try writeToken(builder, pointer_type.op_token, .operator); - break :blk pointer_type.op_token + 1; - }, - .LBracket => blk: { - try writeToken(builder, pointer_type.op_token + 1, .operator); - const is_c_ptr = tok_ids[pointer_type.op_token + 2] == .Identifier; - - if (is_c_ptr) { - try writeToken(builder, pointer_type.op_token + 2, .operator); - } - - if (ptr_info.sentinel) |sentinel| { - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, sentinel }); - break :blk sentinel.lastToken() + 2; - } - - break :blk pointer_type.op_token + 3 + @boolToInt(is_c_ptr); - }, - else => 0, - }; - - if (ptr_info.align_info) |align_info| { - try writeToken(builder, off, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, align_info.node }); - } - try writeToken(builder, ptr_info.const_token, .keyword); - try writeToken(builder, ptr_info.volatile_token, .keyword); - try writeToken(builder, ptr_info.allowzero_token, .keyword); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, rhs }); - }, - .ArrayType => { - const array_type = node.castTag(.ArrayType).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.len_expr }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.rhs }); - }, - .ArrayTypeSentinel => { - const array_type = node.castTag(.ArrayTypeSentinel).?; - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.len_expr }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.sentinel }); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.rhs }); - }, - .AddressOf, .Await, .BitNot, .BoolNot, .OptionalType, .Negation, .NegationWrap, .Resume, .Try => { - const prefix_op = node.cast(ast.Node.SimplePrefixOp).?; - const tok_type: TokenType = switch (node.tag) { - .Try, .Await, .Resume => .keyword, + .add, + .add_wrap, + .array_cat, + .array_mult, + .assign, + .assign_bit_and, + .assign_bit_or, + .assign_bit_shift_left, + .assign_bit_shift_right, + .assign_bit_xor, + .assign_div, + .assign_sub, + .assign_sub_wrap, + .assign_mod, + .assign_add, + .assign_add_wrap, + .assign_mul, + .assign_mul_wrap, + .bang_equal, + .bit_and, + .bit_or, + .bit_shift_left, + .bit_shift_right, + .bit_xor, + .bool_and, + .bool_or, + .div, + .equal_equal, + .error_union, + .greater_or_equal, + .greater_than, + .less_or_equal, + .less_than, + .merge_error_sets, + .mod, + .mul, + .mul_wrap, + .switch_range, + .sub, + .sub_wrap, + .@"orelse", + => { + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); + const token_type: TokenType = switch (tag) { + .bool_and, .bool_or => .keyword, else => .operator, }; - try writeToken(builder, prefix_op.op_token, tok_type); - try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, prefix_op.rhs }); + + try writeToken(builder, main_token, token_type); + if (datas[node].rhs != 0) + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].rhs }); }, - else => {}, + .field_access => { + const data = datas[node]; + if (data.rhs == 0) return; + const rhs_str = tree.tokenSlice(data.rhs); + + // TODO This is basically exactly the same as what is done in analysis.resolveTypeOfNode, with the added + // writeToken code. + // Maybe we can hook into it insead? Also applies to Identifier and VarDecl + var bound_type_params = analysis.BoundTypeParams.init(&arena.allocator); + const lhs_type = try analysis.resolveFieldAccessLhsType( + store, + arena, + (try analysis.resolveTypeOfNodeInternal(store, arena, .{ + .node = data.lhs, + .handle = handle, + }, &bound_type_params)) orelse return, + &bound_type_params, + ); + const left_type_node = switch (lhs_type.type.data) { + .other => |n| n, + else => return, + }; + if (try analysis.lookupSymbolContainer(store, arena, .{ + .node = left_type_node, + .handle = lhs_type.handle, + }, rhs_str, !lhs_type.type.is_type_val)) |decl_type| { + switch (decl_type.decl.*) { + .ast_node => |decl_node| { + if (decl_type.handle.tree.nodes.items(.tag)[decl_node].isContainerField()) { + const tok_type: ?TokenType = if (analysis.isContainer(lhs_type.handle.tree.nodes.items(.tag)[left_type_node])) + fieldTokenType(decl_node, lhs_type.handle) + else if (left_type_node == 0) + TokenType.field + else + null; + + if (tok_type) |tt| try writeToken(builder, data.rhs, tt); + return; + } else if (decl_type.handle.tree.nodes.items(.tag)[decl_node] == .error_value) { + try writeToken(builder, data.rhs, .errorTag); + } + }, + else => {}, + } + + if (try decl_type.resolveType(store, arena, &bound_type_params)) |resolved_type| { + try colorIdentifierBasedOnType(builder, resolved_type, data.rhs, .{}); + } + } + }, + .ptr_type, + .ptr_type_aligned, + .ptr_type_bit_range, + .ptr_type_sentinel, + => { + const ptr_type = analysis.ptrType(tree, node).?; + + if (ptr_type.size == .One and token_tags[main_token] == .asterisk_asterisk and + main_token == main_tokens[ptr_type.ast.child_type]) + { + return try await @asyncCall(child_frame, {}, writeNodeTokens, .{ + builder, + arena, + store, + ptr_type.ast.child_type, + }); + } + + try writeToken(builder, main_token, .operator); + if (ptr_type.ast.sentinel != 0) { + return try await @asyncCall(child_frame, {}, writeNodeTokens, .{ + builder, + arena, + store, + ptr_type.ast.sentinel, + }); + } + + try writeToken(builder, ptr_type.allowzero_token, .keyword); + + if (ptr_type.ast.align_node != 0) { + const first_tok = tree.firstToken(ptr_type.ast.align_node); + try writeToken(builder, first_tok - 2, .keyword); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.align_node }); + + if (ptr_type.ast.bit_range_start != 0) { + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.bit_range_start }); + try writeToken(builder, tree.firstToken(ptr_type.ast.bit_range_end - 1), .operator); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.bit_range_end }); + } + } + + try writeToken(builder, ptr_type.const_token, .keyword); + try writeToken(builder, ptr_type.volatile_token, .keyword); + + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.child_type }); + }, + .array_type, + .array_type_sentinel, + => { + const array_type: ast.full.ArrayType = if (tag == .array_type) + tree.arrayType(node) + else + tree.arrayTypeSentinel(node); + + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.ast.elem_count }); + if (array_type.ast.sentinel) |sentinel| + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, sentinel }); + + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.ast.elem_type }); + }, + .address_of, + .bit_not, + .bool_not, + .optional_type, + .negation, + .negation_wrap, + => { + try writeToken(builder, main_token, .operator); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); + }, + .@"try", + .@"resume", + .@"await", + => { + try writeToken(builder, main_token, .keyword); + try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, datas[node].lhs }); + }, + else => std.log.scoped(.semantic_tokens).debug("TODO: {s}", .{tag}), } } // TODO Range version, edit version. pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentStore, handle: *DocumentStore.Handle, encoding: offsets.Encoding) ![]u32 { var builder = Builder.init(arena.child_allocator, handle, encoding); + // pass root node, which always has index '0' - try writeNodeTokens(&builder, arena, store, 0, handle.tree); + try writeNodeTokens(&builder, arena, store, 0); return builder.toOwnedSlice(); }