Fuzzer fixes (#940)

* better handling of container_decl_arg_trailing

* ignore semantic token when moving backwards

* use custom ast functions instead of from std
This commit is contained in:
Techatrix 2023-01-26 22:04:49 +01:00 committed by GitHub
parent fe54fb7cfa
commit 6019eff13e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 27 deletions

View File

@ -254,29 +254,29 @@ pub fn forFull(tree: Ast, node: Node.Index) full.While {
pub fn fullPtrType(tree: Ast, node: Node.Index) ?full.PtrType { pub fn fullPtrType(tree: Ast, node: Node.Index) ?full.PtrType {
return switch (tree.nodes.items(.tag)[node]) { return switch (tree.nodes.items(.tag)[node]) {
.ptr_type_aligned => tree.ptrTypeAligned(node), .ptr_type_aligned => ptrTypeAligned(tree, node),
.ptr_type_sentinel => tree.ptrTypeSentinel(node), .ptr_type_sentinel => ptrTypeSentinel(tree, node),
.ptr_type => tree.ptrType(node), .ptr_type => ptrTypeSimple(tree, node),
.ptr_type_bit_range => tree.ptrTypeBitRange(node), .ptr_type_bit_range => ptrTypeBitRange(tree, node),
else => null, else => null,
}; };
} }
pub fn fullIf(tree: Ast, node: Node.Index) ?full.If { pub fn fullIf(tree: Ast, node: Node.Index) ?full.If {
return switch (tree.nodes.items(.tag)[node]) { return switch (tree.nodes.items(.tag)[node]) {
.if_simple => tree.ifSimple(node), .if_simple => ifSimple(tree, node),
.@"if" => tree.ifFull(node), .@"if" => ifFull(tree, node),
else => null, else => null,
}; };
} }
pub fn fullWhile(tree: Ast, node: Node.Index) ?full.While { pub fn fullWhile(tree: Ast, node: Node.Index) ?full.While {
return switch (tree.nodes.items(.tag)[node]) { return switch (tree.nodes.items(.tag)[node]) {
.while_simple => tree.whileSimple(node), .while_simple => whileSimple(tree, node),
.while_cont => tree.whileCont(node), .while_cont => whileCont(tree, node),
.@"while" => tree.whileFull(node), .@"while" => whileFull(tree, node),
.for_simple => tree.forSimple(node), .for_simple => forSimple(tree, node),
.@"for" => tree.forFull(node), .@"for" => forFull(tree, node),
else => null, else => null,
}; };
} }
@ -543,7 +543,9 @@ pub fn lastToken(tree: Ast, node: Ast.Node.Index) Ast.TokenIndex {
n = tree.extra_data[cases.end - 1]; // last case n = tree.extra_data[cases.end - 1]; // last case
} }
}, },
.container_decl_arg => { .container_decl_arg,
.container_decl_arg_trailing,
=> {
const members = tree.extraData(datas[n].rhs, Node.SubRange); const members = tree.extraData(datas[n].rhs, Node.SubRange);
if (members.end - members.start == 0) { if (members.end - members.start == 0) {
end_offset += 3; // for the rparen + lbrace + rbrace end_offset += 3; // for the rparen + lbrace + rbrace
@ -567,7 +569,6 @@ pub fn lastToken(tree: Ast, node: Ast.Node.Index) Ast.TokenIndex {
}, },
.array_init_comma, .array_init_comma,
.struct_init_comma, .struct_init_comma,
.container_decl_arg_trailing,
.switch_comma, .switch_comma,
=> { => {
if (datas[n].rhs != 0) { if (datas[n].rhs != 0) {

View File

@ -74,9 +74,7 @@ const Builder = struct {
const starts = tree.tokens.items(.start); const starts = tree.tokens.items(.start);
const next_start = starts[token]; const next_start = starts[token];
if (next_start < self.previous_position) { if (next_start < self.previous_position) return;
return error.MovedBackwards;
}
if (self.previous_token) |prev| { if (self.previous_token) |prev| {
// Highlight gaps between AST nodes. These can contain comments or malformed code. // Highlight gaps between AST nodes. These can contain comments or malformed code.
@ -182,6 +180,8 @@ const Builder = struct {
} }
fn addDirect(self: *Builder, tok_type: TokenType, tok_mod: TokenModifiers, start: usize, length: usize) !void { fn addDirect(self: *Builder, tok_type: TokenType, tok_mod: TokenModifiers, start: usize, length: usize) !void {
if (start < self.previous_position) return;
const text = self.handle.tree.source[self.previous_position..start]; const text = self.handle.tree.source[self.previous_position..start];
const delta = offsets.indexToPosition(text, text.len, self.encoding); const delta = offsets.indexToPosition(text, text.len, self.encoding);
@ -261,13 +261,8 @@ fn colorIdentifierBasedOnType(builder: *Builder, type_node: analysis.TypeWithHan
} }
} }
const WriteTokensError = error{
OutOfMemory,
MovedBackwards,
};
/// HACK self-hosted has not implemented async yet /// HACK self-hosted has not implemented async yet
fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) WriteTokensError!void { fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) error{OutOfMemory}!void {
if (zig_builtin.zig_backend == .other or zig_builtin.zig_backend == .stage1) { if (zig_builtin.zig_backend == .other or zig_builtin.zig_backend == .stage1) {
const FrameSize = @sizeOf(@Frame(writeNodeTokens)); const FrameSize = @sizeOf(@Frame(writeNodeTokens));
var child_frame = try allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize); var child_frame = try allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize);
@ -280,7 +275,7 @@ fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) WriteTokensE
} }
} }
fn writeNodeTokens(builder: *Builder, maybe_node: ?Ast.Node.Index) WriteTokensError!void { fn writeNodeTokens(builder: *Builder, maybe_node: ?Ast.Node.Index) error{OutOfMemory}!void {
const node = maybe_node orelse return; const node = maybe_node orelse return;
const handle = builder.handle; const handle = builder.handle;
@ -1010,10 +1005,7 @@ pub fn writeAllSemanticTokens(
// reverse the ast from the root declarations // reverse the ast from the root declarations
for (handle.tree.rootDecls()) |child| { for (handle.tree.rootDecls()) |child| {
writeNodeTokens(&builder, child) catch |err| switch (err) { try writeNodeTokens(&builder, child);
error.MovedBackwards => break,
else => |e| return e,
};
} }
try builder.finish(); try builder.finish();
return builder.toOwnedSlice(); return builder.toOwnedSlice();

View File

@ -32,6 +32,18 @@ test "semantic tokens - comments" {
// TODO more tests // TODO more tests
} }
test "semantic tokens - string literals" {
// https://github.com/zigtools/zls/issues/921
try testSemanticTokens(
\\"
\\"",//
\\"":
,
// no idea if this output is correct but at least it doesn't crash
&.{ 1, 3, 3, 8, 0, 1, 0, 2, 4, 0, 0, 0, 2, 9, 0 },
);
}
const file_uri = switch (builtin.os.tag) { const file_uri = switch (builtin.os.tag) {
.windows => "file:///C:/test.zig", .windows => "file:///C:/test.zig",
else => "file:///test.zig", else => "file:///test.zig",