From 6019eff13e92412e441770b9488def38c3e127a2 Mon Sep 17 00:00:00 2001 From: Techatrix <19954306+Techatrix@users.noreply.github.com> Date: Thu, 26 Jan 2023 22:04:49 +0100 Subject: [PATCH] Fuzzer fixes (#940) * better handling of container_decl_arg_trailing * ignore semantic token when moving backwards * use custom ast functions instead of from std --- src/ast.zig | 27 +++++++++++++------------- src/semantic_tokens.zig | 20 ++++++------------- tests/lsp_features/semantic_tokens.zig | 12 ++++++++++++ 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/ast.zig b/src/ast.zig index cfbf935..71cdf61 100644 --- a/src/ast.zig +++ b/src/ast.zig @@ -254,29 +254,29 @@ pub fn forFull(tree: Ast, node: Node.Index) full.While { pub fn fullPtrType(tree: Ast, node: Node.Index) ?full.PtrType { return switch (tree.nodes.items(.tag)[node]) { - .ptr_type_aligned => tree.ptrTypeAligned(node), - .ptr_type_sentinel => tree.ptrTypeSentinel(node), - .ptr_type => tree.ptrType(node), - .ptr_type_bit_range => tree.ptrTypeBitRange(node), + .ptr_type_aligned => ptrTypeAligned(tree, node), + .ptr_type_sentinel => ptrTypeSentinel(tree, node), + .ptr_type => ptrTypeSimple(tree, node), + .ptr_type_bit_range => ptrTypeBitRange(tree, node), else => null, }; } pub fn fullIf(tree: Ast, node: Node.Index) ?full.If { return switch (tree.nodes.items(.tag)[node]) { - .if_simple => tree.ifSimple(node), - .@"if" => tree.ifFull(node), + .if_simple => ifSimple(tree, node), + .@"if" => ifFull(tree, node), else => null, }; } pub fn fullWhile(tree: Ast, node: Node.Index) ?full.While { return switch (tree.nodes.items(.tag)[node]) { - .while_simple => tree.whileSimple(node), - .while_cont => tree.whileCont(node), - .@"while" => tree.whileFull(node), - .for_simple => tree.forSimple(node), - .@"for" => tree.forFull(node), + .while_simple => whileSimple(tree, node), + .while_cont => whileCont(tree, node), + .@"while" => whileFull(tree, node), + .for_simple => forSimple(tree, node), + .@"for" => forFull(tree, node), else => null, }; } @@ -543,7 +543,9 @@ pub fn lastToken(tree: Ast, node: Ast.Node.Index) Ast.TokenIndex { n = tree.extra_data[cases.end - 1]; // last case } }, - .container_decl_arg => { + .container_decl_arg, + .container_decl_arg_trailing, + => { const members = tree.extraData(datas[n].rhs, Node.SubRange); if (members.end - members.start == 0) { end_offset += 3; // for the rparen + lbrace + rbrace @@ -567,7 +569,6 @@ pub fn lastToken(tree: Ast, node: Ast.Node.Index) Ast.TokenIndex { }, .array_init_comma, .struct_init_comma, - .container_decl_arg_trailing, .switch_comma, => { if (datas[n].rhs != 0) { diff --git a/src/semantic_tokens.zig b/src/semantic_tokens.zig index 0484971..254f551 100644 --- a/src/semantic_tokens.zig +++ b/src/semantic_tokens.zig @@ -74,9 +74,7 @@ const Builder = struct { const starts = tree.tokens.items(.start); const next_start = starts[token]; - if (next_start < self.previous_position) { - return error.MovedBackwards; - } + if (next_start < self.previous_position) return; if (self.previous_token) |prev| { // Highlight gaps between AST nodes. These can contain comments or malformed code. @@ -182,6 +180,8 @@ const Builder = struct { } fn addDirect(self: *Builder, tok_type: TokenType, tok_mod: TokenModifiers, start: usize, length: usize) !void { + if (start < self.previous_position) return; + const text = self.handle.tree.source[self.previous_position..start]; const delta = offsets.indexToPosition(text, text.len, self.encoding); @@ -261,13 +261,8 @@ fn colorIdentifierBasedOnType(builder: *Builder, type_node: analysis.TypeWithHan } } -const WriteTokensError = error{ - OutOfMemory, - MovedBackwards, -}; - /// HACK self-hosted has not implemented async yet -fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) WriteTokensError!void { +fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) error{OutOfMemory}!void { if (zig_builtin.zig_backend == .other or zig_builtin.zig_backend == .stage1) { const FrameSize = @sizeOf(@Frame(writeNodeTokens)); var child_frame = try allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize); @@ -280,7 +275,7 @@ fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) WriteTokensE } } -fn writeNodeTokens(builder: *Builder, maybe_node: ?Ast.Node.Index) WriteTokensError!void { +fn writeNodeTokens(builder: *Builder, maybe_node: ?Ast.Node.Index) error{OutOfMemory}!void { const node = maybe_node orelse return; const handle = builder.handle; @@ -1010,10 +1005,7 @@ pub fn writeAllSemanticTokens( // reverse the ast from the root declarations for (handle.tree.rootDecls()) |child| { - writeNodeTokens(&builder, child) catch |err| switch (err) { - error.MovedBackwards => break, - else => |e| return e, - }; + try writeNodeTokens(&builder, child); } try builder.finish(); return builder.toOwnedSlice(); diff --git a/tests/lsp_features/semantic_tokens.zig b/tests/lsp_features/semantic_tokens.zig index 459a0ea..f36657d 100644 --- a/tests/lsp_features/semantic_tokens.zig +++ b/tests/lsp_features/semantic_tokens.zig @@ -32,6 +32,18 @@ test "semantic tokens - comments" { // TODO more tests } +test "semantic tokens - string literals" { + // https://github.com/zigtools/zls/issues/921 + try testSemanticTokens( + \\" + \\"",// + \\"": + , + // no idea if this output is correct but at least it doesn't crash + &.{ 1, 3, 3, 8, 0, 1, 0, 2, 4, 0, 0, 0, 2, 9, 0 }, + ); +} + const file_uri = switch (builtin.os.tag) { .windows => "file:///C:/test.zig", else => "file:///test.zig",