sylv/zls
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Self hosted fixes (#682)

Browse Source 
* fix compiler errors when using self-hosted

* move parameters into builder struct

* simplify code when using self hosted

* fix stage2 error `cast discards const qualifier`

* correctly return after finding method

* fix position context tests on self hosted

* remove unnecessary comptime in position context tests
This commit is contained in:
Techatrix
2022-09-30 06:04:55 +02:00
committed by GitHub
parent d030dd0993
commit 51356517a9
9 changed files with 307 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
src/semantic_tokens.zig
View File

@@ -1,4 +1,5 @@
const std = @import("std");
const zig_builtin = @import("builtin");
const offsets = @import("offsets.zig");
const DocumentStore = @import("DocumentStore.zig");
const analysis = @import("analysis.zig");
@@ -50,16 +51,18 @@ pub const TokenModifiers = packed struct {
};
const Builder = struct {
allocator: std.mem.Allocator,
arena: *std.heap.ArenaAllocator,
store: *DocumentStore,
handle: *DocumentStore.Handle,
previous_position: usize = 0,
previous_token: ?Ast.TokenIndex = null,
arr: std.ArrayListUnmanaged(u32),
encoding: offsets.Encoding,
fn init(allocator: std.mem.Allocator, handle: *DocumentStore.Handle, encoding: offsets.Encoding) Builder {
fn init(arena: *std.heap.ArenaAllocator, store: *DocumentStore, handle: *DocumentStore.Handle, encoding: offsets.Encoding) Builder {
return Builder{
.allocator = allocator,
.arena = arena,
.store = store,
.handle = handle,
.arr = std.ArrayListUnmanaged(u32){},
.encoding = encoding,
@@ -182,7 +185,7 @@ const Builder = struct {
const text = self.handle.tree.source[self.previous_position..start];
const delta = offsets.indexToPosition(text, text.len, self.encoding);
try self.arr.appendSlice(self.allocator, &.{
try self.arr.appendSlice(self.arena.allocator(), &.{
@truncate(u32, delta.line),
@truncate(u32, delta.character),
@truncate(u32, length),
@@ -193,7 +196,7 @@ const Builder = struct {
}
fn toOwnedSlice(self: *Builder) []u32 {
return self.arr.toOwnedSlice(self.allocator);
return self.arr.toOwnedSlice(self.arena.allocator());
}
};
@@ -260,16 +263,24 @@ fn colorIdentifierBasedOnType(builder: *Builder, type_node: analysis.TypeWithHan
const WriteTokensError = error{
OutOfMemory,
Utf8InvalidStartByte,
CodepointTooLong,
Utf8ExpectedContinuation,
Utf8OverlongEncoding,
Utf8EncodesSurrogateHalf,
Utf8CodepointTooLarge,
MovedBackwards,
};
fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *DocumentStore, maybe_node: ?Ast.Node.Index) WriteTokensError!void {
/// HACK self-hosted has not implemented async yet
fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) WriteTokensError!void {
if (zig_builtin.zig_backend == .other or zig_builtin.zig_backend == .stage1) {
const FrameSize = @sizeOf(@Frame(writeNodeTokens));
var child_frame = try allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize);
// defer allocator.free(child_frame); allocator is a arena allocator
return await @asyncCall(child_frame, {}, writeNodeTokens, args);
} else {
// TODO find a non recursive solution
return @call(.{}, writeNodeTokens, args);
}
}
fn writeNodeTokens(builder: *Builder, maybe_node: ?Ast.Node.Index) WriteTokensError!void {
const node = maybe_node orelse return;
const handle = builder.handle;
@@ -280,9 +291,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
const main_tokens = tree.nodes.items(.main_token);
if (node == 0 or node > node_data.len) return;
const FrameSize = @sizeOf(@Frame(writeNodeTokens));
var child_frame = try arena.child_allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize);
defer arena.child_allocator.free(child_frame);
var allocator = builder.arena.allocator();
const tag = node_tags[node];
const main_token = main_tokens[node];
@@ -292,7 +301,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
.container_field,
.container_field_align,
.container_field_init,
=> try writeContainerField(builder, arena, store, node, .field, child_frame),
=> try writeContainerField(builder, node, .field),
.@"errdefer" => {
try writeToken(builder, main_token, .keyword);
@@ -303,7 +312,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, payload_tok + 1, .operator);
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.block,
.block_semicolon,
@@ -319,9 +328,9 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
for (statements) |child| {
if (node_tags[child].isContainerField()) {
try writeContainerField(builder, arena, store, child, .field, child_frame);
try writeContainerField(builder, child, .field);
} else {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, child });
try callWriteNodeTokens(allocator, .{ builder, child });
}
}
},
@@ -340,18 +349,18 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, var_decl.comptime_token, .keyword);
try writeToken(builder, var_decl.ast.mut_token, .keyword);
if (try analysis.resolveTypeOfNode(store, arena, .{ .node = node, .handle = handle })) |decl_type| {
if (try analysis.resolveTypeOfNode(builder.store, builder.arena, .{ .node = node, .handle = handle })) |decl_type| {
try colorIdentifierBasedOnType(builder, decl_type, var_decl.ast.mut_token + 1, .{ .declaration = true });
} else {
try writeTokenMod(builder, var_decl.ast.mut_token + 1, .variable, .{ .declaration = true });
}
try writeToken(builder, var_decl.ast.mut_token + 2, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.type_node });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.align_node });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.section_node });
try callWriteNodeTokens(allocator, .{ builder, var_decl.ast.type_node });
try callWriteNodeTokens(allocator, .{ builder, var_decl.ast.align_node });
try callWriteNodeTokens(allocator, .{ builder, var_decl.ast.section_node });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, var_decl.ast.init_node });
try callWriteNodeTokens(allocator, .{ builder, var_decl.ast.init_node });
},
.@"usingnamespace" => {
const first_tok = tree.firstToken(node);
@@ -359,7 +368,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeDocComments(builder, tree, first_tok - 1);
try writeToken(builder, if (token_tags[first_tok] == .keyword_pub) first_tok else null, .keyword);
try writeToken(builder, main_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
},
.container_decl,
.container_decl_trailing,
@@ -381,17 +390,17 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, decl.ast.main_token, .keyword);
if (decl.ast.enum_token) |enum_token| {
if (decl.ast.arg != 0)
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, decl.ast.arg })
try callWriteNodeTokens(allocator, .{ builder, decl.ast.arg })
else
try writeToken(builder, enum_token, .keyword);
} else try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, decl.ast.arg });
} else try callWriteNodeTokens(allocator, .{ builder, decl.ast.arg });
const field_token_type = fieldTokenType(node, handle);
for (decl.ast.members) |child| {
if (node_tags[child].isContainerField()) {
try writeContainerField(builder, arena, store, child, field_token_type, child_frame);
try writeContainerField(builder, child, field_token_type);
} else {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, child });
try callWriteNodeTokens(allocator, .{ builder, child });
}
}
},
@@ -411,8 +420,8 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
}
if (try analysis.lookupSymbolGlobal(
store,
arena,
builder.store,
builder.arena,
handle,
name,
tree.tokens.items(.start)[main_token],
@@ -421,7 +430,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
return try writeToken(builder, main_token, .parameter);
}
var bound_type_params = analysis.BoundTypeParams{};
if (try child.resolveType(store, arena, &bound_type_params)) |decl_type| {
if (try child.resolveType(builder.store, builder.arena, &bound_type_params)) |decl_type| {
try colorIdentifierBasedOnType(builder, decl_type, main_token, .{});
} else {
try writeTokenMod(builder, main_token, .variable, .{});
@@ -464,28 +473,28 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeTokenMod(builder, param_decl.name_token, .parameter, .{ .declaration = true });
if (param_decl.anytype_ellipsis3) |any_token| {
try writeToken(builder, any_token, .type);
} else try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param_decl.type_expr });
} else try callWriteNodeTokens(allocator, .{ builder, param_decl.type_expr });
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.align_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.section_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.callconv_expr });
try callWriteNodeTokens(allocator, .{ builder, fn_proto.ast.align_expr });
try callWriteNodeTokens(allocator, .{ builder, fn_proto.ast.section_expr });
try callWriteNodeTokens(allocator, .{ builder, fn_proto.ast.callconv_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, fn_proto.ast.return_type });
try callWriteNodeTokens(allocator, .{ builder, fn_proto.ast.return_type });
if (tag == .fn_decl)
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.anyframe_type => {
try writeToken(builder, main_token, .type);
if (node_data[node].rhs != 0) {
try writeToken(builder, node_data[node].lhs, .type);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
}
},
.@"defer" => {
try writeToken(builder, main_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.@"comptime",
.@"nosuspend",
@@ -493,25 +502,25 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
if (analysis.getDocCommentTokenIndex(token_tags, main_token)) |doc|
try writeDocComments(builder, tree, doc);
try writeToken(builder, main_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
},
.@"switch",
.switch_comma,
=> {
try writeToken(builder, main_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
const extra = tree.extraData(node_data[node].rhs, Ast.Node.SubRange);
const cases = tree.extra_data[extra.start..extra.end];
for (cases) |case_node| {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, case_node });
try callWriteNodeTokens(allocator, .{ builder, case_node });
}
},
.switch_case_one,
.switch_case,
=> {
const switch_case = if (tag == .switch_case) tree.switchCase(node) else tree.switchCaseOne(node);
for (switch_case.ast.values) |item_node| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, item_node });
for (switch_case.ast.values) |item_node| try callWriteNodeTokens(allocator, .{ builder, item_node });
// check it it's 'else'
if (switch_case.ast.values.len == 0) try writeToken(builder, switch_case.ast.arrow_token - 1, .keyword);
try writeToken(builder, switch_case.ast.arrow_token, .operator);
@@ -519,7 +528,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
const actual_payload = payload_token + @boolToInt(token_tags[payload_token] == .asterisk);
try writeToken(builder, actual_payload, .variable);
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, switch_case.ast.target_expr });
try callWriteNodeTokens(allocator, .{ builder, switch_case.ast.target_expr });
},
.@"while",
.while_simple,
@@ -531,7 +540,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, while_node.label_token, .label);
try writeToken(builder, while_node.inline_token, .keyword);
try writeToken(builder, while_node.ast.while_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.cond_expr });
try callWriteNodeTokens(allocator, .{ builder, while_node.ast.cond_expr });
if (while_node.payload_token) |payload| {
try writeToken(builder, payload - 1, .operator);
try writeToken(builder, payload, .variable);
@@ -543,9 +552,9 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
}
try writeToken(builder, r_pipe, .operator);
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.cont_expr });
try callWriteNodeTokens(allocator, .{ builder, while_node.ast.cont_expr });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.then_expr });
try callWriteNodeTokens(allocator, .{ builder, while_node.ast.then_expr });
if (while_node.ast.else_expr != 0) {
try writeToken(builder, while_node.else_token, .keyword);
@@ -555,7 +564,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, err_token, .variable);
try writeToken(builder, err_token + 1, .operator);
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, while_node.ast.else_expr });
try callWriteNodeTokens(allocator, .{ builder, while_node.ast.else_expr });
}
},
.@"if",
@@ -564,7 +573,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
const if_node = ast.ifFull(tree, node);
try writeToken(builder, if_node.ast.if_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.ast.cond_expr });
try callWriteNodeTokens(allocator, .{ builder, if_node.ast.cond_expr });
if (if_node.payload_token) |payload| {
// if (?x) |x|
@@ -572,7 +581,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, payload, .variable); // x
try writeToken(builder, payload + 1, .operator); // |
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.ast.then_expr });
try callWriteNodeTokens(allocator, .{ builder, if_node.ast.then_expr });
if (if_node.ast.else_expr != 0) {
try writeToken(builder, if_node.else_token, .keyword);
@@ -582,7 +591,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, err_token, .variable); // err
try writeToken(builder, err_token + 1, .operator); // |
}
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, if_node.ast.else_expr });
try callWriteNodeTokens(allocator, .{ builder, if_node.ast.else_expr });
}
},
.array_init,
@@ -603,8 +612,8 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
else => unreachable,
};
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_init.ast.type_expr });
for (array_init.ast.elements) |elem| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, elem });
try callWriteNodeTokens(allocator, .{ builder, array_init.ast.type_expr });
for (array_init.ast.elements) |elem| try callWriteNodeTokens(allocator, .{ builder, elem });
},
.struct_init,
.struct_init_comma,
@@ -627,12 +636,13 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
var field_token_type: ?TokenType = null;
if (struct_init.ast.type_expr != 0) {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, struct_init.ast.type_expr });
try callWriteNodeTokens(allocator, .{ builder, struct_init.ast.type_expr });
field_token_type = if (try analysis.resolveTypeOfNode(store, arena, .{
.node = struct_init.ast.type_expr,
.handle = handle,
})) |struct_type| switch (struct_type.type.data) {
field_token_type = if (try analysis.resolveTypeOfNode(
builder.store,
builder.arena,
.{ .node = struct_init.ast.type_expr, .handle = handle },
)) |struct_type| switch (struct_type.type.data) {
.other => |type_node| if (ast.isContainer(struct_type.handle.tree, type_node))
fieldTokenType(type_node, struct_type.handle)
else
@@ -646,7 +656,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, init_token - 3, field_token_type orelse .field); // '.'
try writeToken(builder, init_token - 2, field_token_type orelse .field); // name
try writeToken(builder, init_token - 1, .operator); // '='
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, field_init });
try callWriteNodeTokens(allocator, .{ builder, field_init });
}
},
.call,
@@ -662,14 +672,14 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
const call = ast.callFull(tree, node, &params).?;
try writeToken(builder, call.async_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, call.ast.fn_expr });
try callWriteNodeTokens(allocator, .{ builder, call.ast.fn_expr });
if (builder.previous_token) |prev| {
if (prev != ast.lastToken(tree, call.ast.fn_expr) and token_tags[ast.lastToken(tree, call.ast.fn_expr)] == .identifier) {
try writeToken(builder, ast.lastToken(tree, call.ast.fn_expr), .function);
}
}
for (call.ast.params) |param| try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param });
for (call.ast.params) |param| try callWriteNodeTokens(allocator, .{ builder, param });
},
.slice,
.slice_open,
@@ -682,27 +692,27 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
else => unreachable,
};
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.sliced });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.start });
try callWriteNodeTokens(allocator, .{ builder, slice.ast.sliced });
try callWriteNodeTokens(allocator, .{ builder, slice.ast.start });
try writeToken(builder, ast.lastToken(tree, slice.ast.start) + 1, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.end });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, slice.ast.sentinel });
try callWriteNodeTokens(allocator, .{ builder, slice.ast.end });
try callWriteNodeTokens(allocator, .{ builder, slice.ast.sentinel });
},
.array_access => {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.deref => {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
try writeToken(builder, main_token, .operator);
},
.unwrap_optional => {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
try writeToken(builder, main_token + 1, .operator);
},
.grouped_expression => {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
},
.@"break",
.@"continue",
@@ -710,11 +720,11 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, main_token, .keyword);
if (node_data[node].lhs != 0)
try writeToken(builder, node_data[node].lhs, .label);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.@"suspend", .@"return" => {
try writeToken(builder, main_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
},
.number_literal => {
try writeToken(builder, main_token, .number);
@@ -733,7 +743,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, main_token, .builtin);
for (params) |param|
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, param });
try callWriteNodeTokens(allocator, .{ builder, param });
},
.string_literal,
.char_literal,
@@ -765,7 +775,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
try writeToken(builder, main_token, .keyword);
try writeToken(builder, asm_node.volatile_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, asm_node.ast.template });
try callWriteNodeTokens(allocator, .{ builder, asm_node.ast.template });
// TODO Inputs, outputs.
},
.test_decl => {
@@ -776,14 +786,14 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
if (token_tags[main_token + 1] == .string_literal)
try writeToken(builder, main_token + 1, .string);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.@"catch" => {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
try writeToken(builder, main_token, .keyword);
if (token_tags[main_token + 1] == .pipe)
try writeToken(builder, main_token + 1, .variable);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.add,
.add_wrap,
@@ -835,43 +845,48 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
.sub_sat,
.@"orelse",
=> {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
const token_type: TokenType = switch (tag) {
.bool_and, .bool_or, .@"orelse" => .keyword,
else => .operator,
};
try writeToken(builder, main_token, token_type);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].rhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].rhs });
},
.field_access => {
const data = node_data[node];
if (data.rhs == 0) return;
const rhs_str = ast.tokenSlice(tree, data.rhs) catch return;
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, data.lhs });
try callWriteNodeTokens(allocator, .{ builder, data.lhs });
// TODO This is basically exactly the same as what is done in analysis.resolveTypeOfNode, with the added
// writeToken code.
// Maybe we can hook into it insead? Also applies to Identifier and VarDecl
var bound_type_params = analysis.BoundTypeParams{};
const lhs_type = try analysis.resolveFieldAccessLhsType(
store,
arena,
(try analysis.resolveTypeOfNodeInternal(store, arena, .{
.node = data.lhs,
.handle = handle,
}, &bound_type_params)) orelse return,
builder.store,
builder.arena,
(try analysis.resolveTypeOfNodeInternal(
builder.store,
builder.arena,
.{ .node = data.lhs, .handle = handle },
&bound_type_params,
)) orelse return,
&bound_type_params,
);
const left_type_node = switch (lhs_type.type.data) {
.other => |n| n,
else => return,
};
if (try analysis.lookupSymbolContainer(store, arena, .{
.node = left_type_node,
.handle = lhs_type.handle,
}, rhs_str, !lhs_type.type.is_type_val)) |decl_type| {
if (try analysis.lookupSymbolContainer(
builder.store,
builder.arena,
.{ .node = left_type_node, .handle = lhs_type.handle },
rhs_str,
!lhs_type.type.is_type_val,
)) |decl_type| {
switch (decl_type.decl.*) {
.ast_node => |decl_node| {
if (decl_type.handle.tree.nodes.items(.tag)[decl_node].isContainerField()) {
@@ -891,7 +906,7 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
else => {},
}
if (try decl_type.resolveType(store, arena, &bound_type_params)) |resolved_type| {
if (try decl_type.resolveType(builder.store, builder.arena, &bound_type_params)) |resolved_type| {
try colorIdentifierBasedOnType(builder, resolved_type, data.rhs, .{});
}
}
@@ -906,12 +921,12 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
if (ptr_type.size == .One and token_tags[main_token] == .asterisk_asterisk and
main_token == main_tokens[ptr_type.ast.child_type])
{
return try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.child_type });
return try callWriteNodeTokens(allocator, .{ builder, ptr_type.ast.child_type });
}
if (ptr_type.size == .One) try writeToken(builder, main_token, .operator);
if (ptr_type.ast.sentinel != 0) {
return try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.sentinel });
return try callWriteNodeTokens(allocator, .{ builder, ptr_type.ast.sentinel });
}
try writeToken(builder, ptr_type.allowzero_token, .keyword);
@@ -919,19 +934,19 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
if (ptr_type.ast.align_node != 0) {
const first_tok = tree.firstToken(ptr_type.ast.align_node);
try writeToken(builder, first_tok - 2, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.align_node });
try callWriteNodeTokens(allocator, .{ builder, ptr_type.ast.align_node });
if (ptr_type.ast.bit_range_start != 0) {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.bit_range_start });
try callWriteNodeTokens(allocator, .{ builder, ptr_type.ast.bit_range_start });
try writeToken(builder, tree.firstToken(ptr_type.ast.bit_range_end - 1), .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.bit_range_end });
try callWriteNodeTokens(allocator, .{ builder, ptr_type.ast.bit_range_end });
}
}
try writeToken(builder, ptr_type.const_token, .keyword);
try writeToken(builder, ptr_type.volatile_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, ptr_type.ast.child_type });
try callWriteNodeTokens(allocator, .{ builder, ptr_type.ast.child_type });
},
.array_type,
.array_type_sentinel,
@@ -941,10 +956,10 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
else
tree.arrayTypeSentinel(node);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.ast.elem_count });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.ast.sentinel });
try callWriteNodeTokens(allocator, .{ builder, array_type.ast.elem_count });
try callWriteNodeTokens(allocator, .{ builder, array_type.ast.sentinel });
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, array_type.ast.elem_type });
try callWriteNodeTokens(allocator, .{ builder, array_type.ast.elem_type });
},
.address_of,
.bit_not,
@@ -954,25 +969,27 @@ fn writeNodeTokens(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *D
.negation_wrap,
=> {
try writeToken(builder, main_token, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
},
.@"try",
.@"resume",
.@"await",
=> {
try writeToken(builder, main_token, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, node_data[node].lhs });
try callWriteNodeTokens(allocator, .{ builder, node_data[node].lhs });
},
.anyframe_literal => try writeToken(builder, main_token, .keyword),
}
}
fn writeContainerField(builder: *Builder, arena: *std.heap.ArenaAllocator, store: *DocumentStore, node: Ast.Node.Index, field_token_type: ?TokenType, child_frame: anytype) !void {
fn writeContainerField(builder: *Builder, node: Ast.Node.Index, field_token_type: ?TokenType) !void {
const tree = builder.handle.tree;
const container_field = ast.containerField(tree, node).?;
const base = tree.nodes.items(.main_token)[node];
const tokens = tree.tokens.items(.tag);
var allocator = builder.arena.allocator();
if (analysis.getDocCommentTokenIndex(tokens, base)) |docs|
try writeDocComments(builder, tree, docs);
@@ -980,10 +997,10 @@ fn writeContainerField(builder: *Builder, arena: *std.heap.ArenaAllocator, store
if (field_token_type) |tok_type| try writeToken(builder, container_field.ast.name_token, tok_type);
if (container_field.ast.type_expr != 0) {
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.ast.type_expr });
try callWriteNodeTokens(allocator, .{ builder, container_field.ast.type_expr });
if (container_field.ast.align_expr != 0) {
try writeToken(builder, tree.firstToken(container_field.ast.align_expr) - 2, .keyword);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.ast.align_expr });
try callWriteNodeTokens(allocator, .{ builder, container_field.ast.align_expr });
}
}
@@ -996,19 +1013,18 @@ fn writeContainerField(builder: *Builder, arena: *std.heap.ArenaAllocator, store
break :block;
try writeToken(builder, eq_tok, .operator);
try await @asyncCall(child_frame, {}, writeNodeTokens, .{ builder, arena, store, container_field.ast.value_expr });
try callWriteNodeTokens(allocator, .{ builder, container_field.ast.value_expr });
}
}
// TODO Range version, edit version.
pub fn writeAllSemanticTokens(arena: *std.heap.ArenaAllocator, store: *DocumentStore, handle: *DocumentStore.Handle, encoding: offsets.Encoding) ![]u32 {
var builder = Builder.init(arena.child_allocator, handle, encoding);
errdefer builder.arr.deinit(arena.child_allocator);
var builder = Builder.init(arena, store, handle, encoding);
// reverse the ast from the root declarations
var buf: [2]Ast.Node.Index = undefined;
for (ast.declMembers(handle.tree, 0, &buf)) |child| {
writeNodeTokens(&builder, arena, store, child) catch |err| switch (err) {
writeNodeTokens(&builder, child) catch |err| switch (err) {
error.MovedBackwards => break,
else => |e| return e,
};