zls/src/semantic_tokens.zig

731 lines
37 KiB
Zig
Raw Normal View History

const std = @import("std");
const offsets = @import("offsets.zig");
const DocumentStore = @import("document_store.zig");
const analysis = @import("analysis.zig");
const ast = std.zig.ast;
const TokenType = enum(u32) {
2020-06-27 18:45:58 +01:00
namespace,
type,
@"struct",
@"enum",
@"union",
parameter,
variable,
tagField,
field,
errorTag,
function,
keyword,
comment,
string,
number,
operator,
2020-06-13 23:07:57 +01:00
builtin,
label,
};
const TokenModifiers = packed struct {
definition: bool = false,
@"async": bool = false,
documentation: bool = false,
2020-06-16 16:49:31 +01:00
generic: bool = false,
fn toInt(self: TokenModifiers) u32 {
2020-06-16 16:49:31 +01:00
return @as(u32, @bitCast(u4, self));
}
2020-06-16 16:49:31 +01:00
inline fn set(self: *TokenModifiers, comptime field: []const u8) void {
@field(self, field) = true;
}
};
const Builder = struct {
handle: *DocumentStore.Handle,
current_token: ?ast.TokenIndex,
arr: std.ArrayList(u32),
encoding: offsets.Encoding,
fn init(allocator: *std.mem.Allocator, handle: *DocumentStore.Handle, encoding: offsets.Encoding) Builder {
return Builder{
.handle = handle,
.current_token = null,
.arr = std.ArrayList(u32).init(allocator),
.encoding = encoding,
};
}
2020-06-13 23:07:57 +01:00
fn add(self: *Builder, token: ast.TokenIndex, token_type: TokenType, token_modifiers: TokenModifiers) !void {
const start_idx = if (self.current_token) |current_token|
self.handle.tree.token_locs[current_token].start
2020-06-13 23:07:57 +01:00
else
0;
const delta_loc = offsets.tokenRelativeLocation(self.handle.tree, start_idx, token, self.encoding) catch return;
2020-06-13 23:07:57 +01:00
try self.arr.appendSlice(&[_]u32{
@truncate(u32, delta_loc.line),
@truncate(u32, delta_loc.column),
@truncate(u32, offsets.tokenLength(self.handle.tree, token, self.encoding)),
2020-06-13 23:07:57 +01:00
@enumToInt(token_type),
token_modifiers.toInt(),
});
self.current_token = token;
}
2020-06-13 23:07:57 +01:00
fn toOwnedSlice(self: *Builder) []u32 {
return self.arr.toOwnedSlice();
}
};
inline fn writeToken(builder: *Builder, token_idx: ?ast.TokenIndex, tok_type: TokenType) !void {
return try writeTokenMod(builder, token_idx, tok_type, .{});
}
inline fn writeTokenMod(builder: *Builder, token_idx: ?ast.TokenIndex, tok_type: TokenType, tok_mod: TokenModifiers) !void {
if (token_idx) |ti| {
try builder.add(ti, tok_type, tok_mod);
2020-06-13 23:07:57 +01:00
}
}
fn writeDocComments(builder: *Builder, tree: *ast.Tree, doc: *ast.Node.DocComment) !void {
var tok_idx = doc.first_line;
while (tree.token_ids[tok_idx] == .DocComment or
tree.token_ids[tok_idx] == .ContainerDocComment or
tree.token_ids[tok_idx] == .LineComment) : (tok_idx += 1)
{
var tok_mod = TokenModifiers{};
if (tree.token_ids[tok_idx] == .DocComment or tree.token_ids[tok_idx] == .ContainerDocComment)
tok_mod.set("documentation");
try builder.add(tok_idx, .comment, tok_mod);
}
}
2020-06-13 23:07:57 +01:00
fn fieldTokenType(container_decl: *ast.Node.ContainerDecl, handle: *DocumentStore.Handle) ?TokenType {
2020-06-18 18:30:58 +01:00
if (container_decl.kind_token > handle.tree.token_ids.len) return null;
return @as(?TokenType, switch (handle.tree.token_ids[container_decl.kind_token]) {
.Keyword_struct => .field,
.Keyword_union, .Keyword_enum => .tagField,
else => null,
});
}
2020-06-18 13:12:09 +01:00
/// This is used to highlight gaps between AST nodes.
/// These gaps can be just gaps between statements/declarations with comments inside them
/// Or malformed code.
const GapHighlighter = struct {
builder: *Builder,
current_idx: ast.TokenIndex,
// TODO More highlighting here
fn handleTok(self: *GapHighlighter, tok: ast.TokenIndex) !void {
const tok_id = self.builder.handle.tree.token_ids[tok];
if (tok_id == .LineComment) {
try writeToken(self.builder, tok, .comment);
2020-06-26 01:26:09 +01:00
} else if (tok_id == .ContainerDocComment) {
try writeTokenMod(self.builder, tok, .comment, TokenModifiers{ .documentation = true });
2020-06-18 13:12:09 +01:00
} else if (@enumToInt(tok_id) >= @enumToInt(std.zig.Token.Id.Keyword_align) and
@enumToInt(tok_id) <= @enumToInt(std.zig.Token.Id.Keyword_while))
{
try writeToken(self.builder, tok, .keyword);
} else if (@enumToInt(tok_id) >= @enumToInt(std.zig.Token.Id.Bang) and
@enumToInt(tok_id) <= @enumToInt(std.zig.Token.Id.Tilde) and
tok_id != .Period and tok_id != .Comma and tok_id != .RParen and
tok_id != .LParen and tok_id != .RBrace and tok_id != .LBrace and
tok_id != .Semicolon and tok_id != .Colon)
{
try writeToken(self.builder, tok, .operator);
} else if (tok_id == .IntegerLiteral or tok_id == .FloatLiteral) {
try writeToken(self.builder, tok, .number);
} else if (tok_id == .StringLiteral or tok_id == .MultilineStringLiteralLine or tok_id == .CharLiteral) {
try writeToken(self.builder, tok, .string);
}
}
fn init(builder: *Builder, start: ast.TokenIndex) GapHighlighter {
return .{ .builder = builder, .current_idx = start };
}
fn next(self: *GapHighlighter, node: *ast.Node) !void {
2020-06-26 12:29:59 +01:00
if (self.current_idx > 0 and self.builder.handle.tree.token_ids[self.current_idx - 1] == .ContainerDocComment) {
try self.handleTok(self.current_idx - 1);
}
2020-06-18 13:12:09 +01:00
var i = self.current_idx;
while (i < node.firstToken()) : (i += 1) {
try self.handleTok(i);
}
self.current_idx = node.lastToken() + 1;
}
fn end(self: *GapHighlighter, last: ast.TokenIndex) !void {
var i = self.current_idx;
while (i < last) : (i += 1) {
try self.handleTok(i);
}
}
};
fn colorIdentifierBasedOnType(builder: *Builder, type_node: analysis.TypeWithHandle, target_tok: ast.TokenIndex, tok_mod: TokenModifiers) !void {
if (type_node.type.is_type_val) {
2020-06-27 18:45:58 +01:00
const tok_type = if (type_node.isNamespace())
.namespace
else if (type_node.isStructType())
2020-06-18 13:12:09 +01:00
.@"struct"
else if (type_node.isEnumType())
.@"enum"
else if (type_node.isUnionType())
.@"union"
else
TokenType.type;
try writeTokenMod(builder, target_tok, tok_type, tok_mod);
} else if (type_node.isTypeFunc()) {
try writeTokenMod(builder, target_tok, .type, tok_mod);
} else if (type_node.isFunc()) {
2020-06-27 18:45:58 +01:00
var new_tok_mod = tok_mod;
if (type_node.isGenericFunc()) {
new_tok_mod.set("generic");
}
try writeTokenMod(builder, target_tok, .function, new_tok_mod);
2020-06-18 13:12:09 +01:00
} else {
try writeTokenMod(builder, target_tok, .variable, tok_mod);
}
}
fn writeContainerField(
builder: *Builder,
arena: *std.heap.ArenaAllocator,
store: *DocumentStore,
container_field: *ast.Node.ContainerField,
field_token_type: ?TokenType,
2020-07-12 20:12:09 +01:00
child_frame: anytype,
) !void {
if (container_field.doc_comments) |docs| try writeDocComments(builder, builder.handle.tree, docs);
try writeToken(builder, container_field.comptime_token, .keyword);
if (field_token_type) |tok_type| try writeToken(builder, container_field.name_token, tok_type);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.type_expr });
2020-06-18 18:30:58 +01:00
if (container_field.align_expr) |n| {
try writeToken(builder, n.firstToken() - 2, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, n });
2020-06-18 18:30:58 +01:00
}
2020-06-18 18:30:58 +01:00
if (container_field.value_expr) |value_expr| block: {
const eq_tok: ast.TokenIndex = if (container_field.type_expr) |type_expr|
type_expr.lastToken() + 1
else if (container_field.align_expr) |align_expr|
align_expr.lastToken() + 1
else
2020-06-18 18:30:58 +01:00
break :block; // Check this, I believe it is correct.
try writeToken(builder, eq_tok, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, value_expr });
}
}
2020-06-18 13:14:55 +01:00
// TODO This is very slow and does a lot of extra work, improve in the future.
fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *DocumentStore, maybe_node: ?*ast.Node) error{OutOfMemory}!void {
if (maybe_node == null) return;
const node = maybe_node.?;
const handle = builder.handle;
const FrameSize = @sizeOf(@Frame(writeNodeTokens));
var child_frame = try arena.child_allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize);
defer arena.child_allocator.free(child_frame);
switch (node.id) {
.Root, .Block => {
2020-06-18 13:12:09 +01:00
const first_tok = if (node.cast(ast.Node.Block)) |block_node| block: {
try writeToken(builder, block_node.label, .label);
2020-06-18 13:12:09 +01:00
break :block block_node.lbrace + 1;
} else 0;
2020-06-18 13:12:09 +01:00
var gap_highlighter = GapHighlighter.init(builder, first_tok);
var child_idx: usize = 0;
while (node.iterate(child_idx)) |child| : (child_idx += 1) {
2020-06-18 13:12:09 +01:00
try gap_highlighter.next(child);
if (child.cast(ast.Node.ContainerField)) |container_field| {
try writeContainerField(builder, arena, store, container_field, .field, child_frame);
} else {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, child });
}
2020-06-17 22:13:56 +01:00
}
2020-06-18 13:12:09 +01:00
try gap_highlighter.end(node.lastToken());
},
.VarDecl => {
const var_decl = node.cast(ast.Node.VarDecl).?;
if (var_decl.doc_comments) |doc| try writeDocComments(builder, handle.tree, doc);
try writeToken(builder, var_decl.visib_token, .keyword);
try writeToken(builder, var_decl.extern_export_token, .keyword);
try writeToken(builder, var_decl.thread_local_token, .keyword);
try writeToken(builder, var_decl.comptime_token, .keyword);
try writeToken(builder, var_decl.mut_token, .keyword);
if (try analysis.resolveTypeOfNode(store, arena, .{ .node = node, .handle = handle })) |decl_type| {
2020-06-18 13:12:09 +01:00
try colorIdentifierBasedOnType(builder, decl_type, var_decl.name_token, .{ .definition = true });
} else {
try writeTokenMod(builder, var_decl.name_token, .variable, .{ .definition = true });
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.type_node });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.align_node });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.section_node });
try writeToken(builder, var_decl.eq_token, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.init_node });
},
.Use => {
const use = node.cast(ast.Node.Use).?;
if (use.doc_comments) |docs| try writeDocComments(builder, builder.handle.tree, docs);
try writeToken(builder, use.visib_token, .keyword);
try writeToken(builder, use.use_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, use.expr });
},
.ErrorSetDecl => {
const error_set = node.cast(ast.Node.ErrorSetDecl).?;
try writeToken(builder, error_set.error_token, .keyword);
for (error_set.declsConst()) |decl|
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, decl });
},
.ContainerDecl => {
const container_decl = node.cast(ast.Node.ContainerDecl).?;
try writeToken(builder, container_decl.layout_token, .keyword);
try writeToken(builder, container_decl.kind_token, .keyword);
switch (container_decl.init_arg_expr) {
.None => {},
.Enum => |enum_expr| if (enum_expr) |expr|
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, expr })
else
try writeToken(builder, container_decl.kind_token + 2, .keyword),
.Type => |type_node| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, type_node }),
}
2020-06-18 13:12:09 +01:00
var gap_highlighter = GapHighlighter.init(builder, container_decl.lbrace_token + 1);
const field_token_type = fieldTokenType(container_decl, handle);
for (container_decl.fieldsAndDeclsConst()) |child| {
2020-06-18 13:12:09 +01:00
try gap_highlighter.next(child);
if (child.cast(ast.Node.ContainerField)) |container_field| {
try writeContainerField(builder, arena, store, container_field, field_token_type, child_frame);
} else {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, child });
}
}
2020-06-18 13:12:09 +01:00
try gap_highlighter.end(node.lastToken());
},
.ErrorTag => {
const error_tag = node.cast(ast.Node.ErrorTag).?;
if (error_tag.doc_comments) |docs| try writeDocComments(builder, handle.tree, docs);
try writeToken(builder, error_tag.firstToken(), .errorTag);
},
.Identifier => {
if (analysis.isTypeIdent(handle.tree, node.firstToken())) {
return try writeToken(builder, node.firstToken(), .type);
}
if (try analysis.lookupSymbolGlobal(store, arena, handle, handle.tree.getNodeSource(node), handle.tree.token_locs[node.firstToken()].start)) |child| {
if (child.decl.* == .param_decl) {
return try writeToken(builder, node.firstToken(), .parameter);
}
2020-06-18 01:23:56 +01:00
var bound_type_params = analysis.BoundTypeParams.init(&arena.allocator);
if (try child.resolveType(store, arena, &bound_type_params)) |decl_type| {
2020-06-18 13:12:09 +01:00
try colorIdentifierBasedOnType(builder, decl_type, node.firstToken(), .{});
} else {
try writeTokenMod(builder, node.firstToken(), .variable, .{});
2020-06-18 01:23:56 +01:00
}
}
},
.FnProto => {
const fn_proto = node.cast(ast.Node.FnProto).?;
if (fn_proto.doc_comments) |docs| try writeDocComments(builder, handle.tree, docs);
try writeToken(builder, fn_proto.visib_token, .keyword);
try writeToken(builder, fn_proto.extern_export_inline_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.lib_name });
try writeToken(builder, fn_proto.fn_token, .keyword);
const func_name_tok_type: TokenType = if (analysis.isTypeFunction(handle.tree, fn_proto))
.type
else
.function;
2020-06-27 18:45:58 +01:00
const tok_mod = if (analysis.isGenericFunction(handle.tree, fn_proto))
TokenModifiers{ .generic = true }
else
TokenModifiers{};
try writeTokenMod(builder, fn_proto.name_token, func_name_tok_type, tok_mod);
for (fn_proto.paramsConst()) |param_decl| {
if (param_decl.doc_comments) |docs| try writeDocComments(builder, handle.tree, docs);
try writeToken(builder, param_decl.noalias_token, .keyword);
try writeToken(builder, param_decl.comptime_token, .keyword);
try writeTokenMod(builder, param_decl.name_token, .parameter, .{ .definition = true });
switch (param_decl.param_type) {
2020-07-12 20:12:09 +01:00
.any_type => |var_node| try writeToken(builder, var_node.firstToken(), .type),
.var_args => |var_args_tok| try writeToken(builder, var_args_tok, .operator),
.type_expr => |type_expr| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, type_expr }),
}
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.align_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.section_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.callconv_expr });
switch (fn_proto.return_type) {
.Explicit => |type_expr| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, type_expr }),
.InferErrorSet => |type_expr| {
try writeToken(builder, type_expr.firstToken() - 1, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, type_expr });
},
.Invalid => {},
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.body_node });
},
.AnyFrameType => {
const any_frame_type = node.cast(ast.Node.AnyFrameType).?;
try writeToken(builder, any_frame_type.anyframe_token, .type);
if (any_frame_type.result) |any_frame_result| {
try writeToken(builder, any_frame_result.arrow_token, .type);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, any_frame_result.return_type });
}
},
.Defer => {
const defer_node = node.cast(ast.Node.Defer).?;
try writeToken(builder, defer_node.defer_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, defer_node.payload });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, defer_node.expr });
},
.Comptime => {
const comptime_node = node.cast(ast.Node.Comptime).?;
if (comptime_node.doc_comments) |docs| try writeDocComments(builder, handle.tree, docs);
try writeToken(builder, comptime_node.comptime_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, comptime_node.expr });
},
.Nosuspend => {
const nosuspend_node = node.cast(ast.Node.Nosuspend).?;
try writeToken(builder, nosuspend_node.nosuspend_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, nosuspend_node.expr });
},
.Payload => {
const payload = node.cast(ast.Node.Payload).?;
try writeToken(builder, payload.lpipe, .operator);
2020-06-18 20:53:33 +01:00
try writeToken(builder, payload.error_symbol.firstToken(), .variable);
try writeToken(builder, payload.rpipe, .operator);
},
.PointerPayload => {
const payload = node.cast(ast.Node.PointerPayload).?;
try writeToken(builder, payload.lpipe, .operator);
try writeToken(builder, payload.ptr_token, .operator);
2020-06-18 20:53:33 +01:00
try writeToken(builder, payload.value_symbol.firstToken(), .variable);
try writeToken(builder, payload.rpipe, .operator);
},
.PointerIndexPayload => {
const payload = node.cast(ast.Node.PointerIndexPayload).?;
try writeToken(builder, payload.lpipe, .operator);
try writeToken(builder, payload.ptr_token, .operator);
2020-06-18 20:53:33 +01:00
try writeToken(builder, payload.value_symbol.firstToken(), .variable);
if (payload.index_symbol) |index_symbol| try writeToken(builder, index_symbol.firstToken(), .variable);
try writeToken(builder, payload.rpipe, .operator);
},
.Else => {
const else_node = node.cast(ast.Node.Else).?;
try writeToken(builder, else_node.else_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, else_node.payload });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, else_node.body });
},
.Switch => {
const switch_node = node.cast(ast.Node.Switch).?;
try writeToken(builder, switch_node.switch_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, switch_node.expr });
2020-06-18 13:12:09 +01:00
var gap_highlighter = GapHighlighter.init(builder, switch_node.expr.lastToken() + 3);
for (switch_node.casesConst()) |case_node| {
2020-06-18 13:12:09 +01:00
try gap_highlighter.next(case_node);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, case_node });
}
2020-06-18 13:12:09 +01:00
try gap_highlighter.end(node.lastToken());
},
.SwitchCase => {
const switch_case = node.cast(ast.Node.SwitchCase).?;
for (switch_case.itemsConst()) |item_node| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, item_node });
try writeToken(builder, switch_case.arrow_token, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, switch_case.payload });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, switch_case.expr });
},
.SwitchElse => {
const switch_else = node.cast(ast.Node.SwitchElse).?;
try writeToken(builder, switch_else.token, .keyword);
},
.While => {
const while_node = node.cast(ast.Node.While).?;
try writeToken(builder, while_node.label, .label);
try writeToken(builder, while_node.inline_token, .keyword);
try writeToken(builder, while_node.while_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.condition });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.payload });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.continue_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.body });
if (while_node.@"else") |else_node|
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, &else_node.base });
},
.For => {
const for_node = node.cast(ast.Node.For).?;
try writeToken(builder, for_node.label, .label);
try writeToken(builder, for_node.inline_token, .keyword);
try writeToken(builder, for_node.for_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, for_node.array_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, for_node.payload });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, for_node.body });
if (for_node.@"else") |else_node|
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, &else_node.base });
},
.If => {
const if_node = node.cast(ast.Node.If).?;
try writeToken(builder, if_node.if_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.condition });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.payload });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.body });
if (if_node.@"else") |else_node|
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, &else_node.base });
},
.InfixOp => {
const infix_op = node.cast(ast.Node.InfixOp).?;
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, infix_op.lhs });
if (infix_op.op != .Period and infix_op.op != .Catch) {
const token_type: TokenType = switch (infix_op.op) {
2020-06-18 13:12:09 +01:00
.BoolAnd, .BoolOr, .UnwrapOptional => .keyword,
else => .operator,
};
try writeToken(builder, infix_op.op_token, token_type);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, infix_op.rhs });
}
2020-06-18 13:12:09 +01:00
switch (infix_op.op) {
.Catch => |n| {
try writeToken(builder, infix_op.op_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, n });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, infix_op.rhs });
2020-06-18 13:12:09 +01:00
},
.Period => {
const rhs_str = handle.tree.tokenSlice(infix_op.rhs.firstToken());
// TODO This is basically exactly the same as what is done in analysis.resolveTypeOfNode, with the added
// writeToken code.
// Maybe we can hook into it insead? Also applies to Identifier and VarDecl
var bound_type_params = analysis.BoundTypeParams.init(&arena.allocator);
const lhs_type = try analysis.resolveFieldAccessLhsType(
store,
arena,
(try analysis.resolveTypeOfNodeInternal(store, arena, .{
.node = infix_op.lhs,
.handle = handle,
}, &bound_type_params)) orelse return,
&bound_type_params,
);
const left_type_node = switch (lhs_type.type.data) {
.other => |n| n,
else => return,
};
if (try analysis.lookupSymbolContainer(store, arena, .{ .node = left_type_node, .handle = lhs_type.handle }, rhs_str, !lhs_type.type.is_type_val)) |decl_type| {
switch (decl_type.decl.*) {
.ast_node => |decl_node| {
if (decl_node.id == .ContainerField) {
const tok_type: ?TokenType = if (left_type_node.cast(ast.Node.ContainerDecl)) |container_decl|
fieldTokenType(container_decl, lhs_type.handle)
else if (left_type_node.id == .Root)
TokenType.field
else
null;
if (tok_type) |tt| try writeToken(builder, infix_op.rhs.firstToken(), tt);
return;
2020-06-18 13:43:03 +01:00
} else if (decl_node.id == .ErrorTag) {
try writeToken(builder, infix_op.rhs.firstToken(), .errorTag);
2020-06-18 13:12:09 +01:00
}
},
else => {},
}
if (try decl_type.resolveType(store, arena, &bound_type_params)) |resolved_type| {
try colorIdentifierBasedOnType(builder, resolved_type, infix_op.rhs.firstToken(), .{});
}
}
},
else => {},
}
},
.PrefixOp => {
const prefix_op = node.cast(ast.Node.PrefixOp).?;
const tok_type: TokenType = switch (prefix_op.op) {
.Try, .Await, .Resume => .keyword,
else => .operator,
};
2020-06-17 22:13:56 +01:00
switch (prefix_op.op) {
.ArrayType => |info| {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, info.len_expr });
2020-06-17 22:13:56 +01:00
},
.SliceType, .PtrType => |info| {
2020-06-18 00:52:03 +01:00
if (prefix_op.op == .PtrType) try writeToken(builder, prefix_op.op_token, tok_type);
2020-06-18 19:31:52 +01:00
if (info.align_info) |align_info| {
if (prefix_op.op == .PtrType) {
try writeToken(builder, prefix_op.op_token + 1, .keyword);
} else {
try writeToken(builder, prefix_op.op_token + 2, .keyword);
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, align_info.node });
2020-06-18 19:31:52 +01:00
}
2020-06-17 22:13:56 +01:00
try writeToken(builder, info.const_token, .keyword);
try writeToken(builder, info.volatile_token, .keyword);
try writeToken(builder, info.allowzero_token, .keyword);
},
else => try writeToken(builder, prefix_op.op_token, tok_type),
2020-06-17 22:13:56 +01:00
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, prefix_op.rhs });
},
.ArrayInitializer => {
const array_initializer = node.cast(ast.Node.ArrayInitializer).?;
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_initializer.lhs });
for (array_initializer.listConst()) |elem| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, elem });
},
.ArrayInitializerDot => {
const array_initializer = node.cast(ast.Node.ArrayInitializerDot).?;
for (array_initializer.listConst()) |elem| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, elem });
},
.StructInitializer => {
const struct_initializer = node.cast(ast.Node.StructInitializer).?;
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, struct_initializer.lhs });
const field_token_type = if (try analysis.resolveTypeOfNode(store, arena, .{ .node = struct_initializer.lhs, .handle = handle })) |struct_type| switch (struct_type.type.data) {
.other => |type_node| if (type_node.cast(ast.Node.ContainerDecl)) |container_decl|
fieldTokenType(container_decl, handle)
else
null,
else => null,
} else null;
2020-06-18 18:30:58 +01:00
var gap_highlighter = GapHighlighter.init(builder, struct_initializer.lhs.lastToken() + 1);
for (struct_initializer.listConst()) |field_init_node| {
2020-06-18 18:30:58 +01:00
try gap_highlighter.next(field_init_node);
std.debug.assert(field_init_node.id == .FieldInitializer);
const field_init = field_init_node.cast(ast.Node.FieldInitializer).?;
if (field_token_type) |tok_type| {
try writeToken(builder, field_init.period_token, tok_type);
try writeToken(builder, field_init.name_token, tok_type);
}
try writeToken(builder, field_init.name_token + 1, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, field_init.expr });
}
2020-06-18 18:30:58 +01:00
try gap_highlighter.end(struct_initializer.rtoken);
},
.StructInitializerDot => {
const struct_initializer = node.cast(ast.Node.StructInitializerDot).?;
2020-06-18 18:30:58 +01:00
var gap_highlighter = GapHighlighter.init(builder, struct_initializer.dot + 1);
for (struct_initializer.listConst()) |field_init_node| {
2020-06-18 18:30:58 +01:00
try gap_highlighter.next(field_init_node);
std.debug.assert(field_init_node.id == .FieldInitializer);
const field_init = field_init_node.cast(ast.Node.FieldInitializer).?;
try writeToken(builder, field_init.period_token, .field);
try writeToken(builder, field_init.name_token, .field);
try writeToken(builder, field_init.name_token + 1, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, field_init.expr });
}
2020-06-18 18:30:58 +01:00
try gap_highlighter.end(struct_initializer.rtoken);
},
.Call => {
const call = node.cast(ast.Node.Call).?;
try writeToken(builder, call.async_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, call.lhs });
2020-06-27 18:45:58 +01:00
if (builder.current_token) |curr_tok| {
if (curr_tok != call.lhs.lastToken() and handle.tree.token_ids[call.lhs.lastToken()] == .Identifier) {
2020-06-27 18:45:58 +01:00
try writeToken(builder, call.lhs.lastToken(), .function);
}
}
for (call.paramsConst()) |param| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param });
},
.SuffixOp => {
const suffix_op = node.cast(ast.Node.SuffixOp).?;
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, suffix_op.lhs });
switch (suffix_op.op) {
.ArrayAccess => |n| {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, n });
},
.Slice => |s| {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, s.start });
try writeToken(builder, s.start.lastToken() + 1, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, s.end });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, s.sentinel });
},
else => try writeToken(builder, suffix_op.rtoken, .operator),
}
},
.GroupedExpression => {
const grouped_expr = node.cast(ast.Node.GroupedExpression).?;
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, grouped_expr.expr });
},
.ControlFlowExpression => {
const cfe = node.cast(ast.Node.ControlFlowExpression).?;
try writeToken(builder, cfe.ltoken, .keyword);
switch (cfe.kind) {
.Break => |label| if (label) |n| try writeToken(builder, n.firstToken(), .label),
.Continue => |label| if (label) |n| try writeToken(builder, n.firstToken(), .label),
else => {},
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, cfe.rhs });
},
.Suspend => {
const suspend_node = node.cast(ast.Node.Suspend).?;
try writeToken(builder, suspend_node.suspend_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, suspend_node.body });
},
.IntegerLiteral => {
try writeToken(builder, node.firstToken(), .number);
},
.EnumLiteral => {
const enum_literal = node.cast(ast.Node.EnumLiteral).?;
try writeToken(builder, enum_literal.dot, .tagField);
try writeToken(builder, enum_literal.name, .tagField);
},
.FloatLiteral => {
try writeToken(builder, node.firstToken(), .number);
},
.BuiltinCall => {
const builtin_call = node.cast(ast.Node.BuiltinCall).?;
try writeToken(builder, builtin_call.builtin_token, .builtin);
for (builtin_call.paramsConst()) |param|
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param });
},
.StringLiteral, .CharLiteral => {
try writeToken(builder, node.firstToken(), .string);
},
.MultilineStringLiteral => {
const multi_line = node.cast(ast.Node.MultilineStringLiteral).?;
for (multi_line.linesConst()) |line| try writeToken(builder, line, .string);
},
.BoolLiteral, .NullLiteral, .UndefinedLiteral, .Unreachable, .ErrorType => {
try writeToken(builder, node.firstToken(), .keyword);
},
.Asm => {
const asm_expr = node.cast(ast.Node.Asm).?;
try writeToken(builder, asm_expr.asm_token, .keyword);
try writeToken(builder, asm_expr.volatile_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, asm_expr.template });
2020-06-17 22:13:56 +01:00
// TODO Inputs, outputs.
},
2020-07-12 20:12:09 +01:00
.AnyType => {
try writeToken(builder, node.firstToken(), .type);
},
.TestDecl => {
const test_decl = node.cast(ast.Node.TestDecl).?;
if (test_decl.doc_comments) |doc| try writeDocComments(builder, handle.tree, doc);
try writeToken(builder, test_decl.test_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, test_decl.name });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, test_decl.body_node });
},
else => {},
2020-06-13 23:07:57 +01:00
}
}
// TODO Range version, edit version.
pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentStore, handle: *DocumentStore.Handle, encoding: offsets.Encoding) ![]u32 {
var builder = Builder.init(arena.child_allocator, handle, encoding);
try writeNodeTokens(&builder, arena, store, &handle.tree.root_node.base);
return builder.toOwnedSlice();
}