Fuzzer fixes (#940)

* better handling of container_decl_arg_trailing

* ignore semantic token when moving backwards

* use custom ast functions instead of from std

src/ast.zig

@@ -254,29 +254,29 @@ pub fn forFull(tree: Ast, node: Node.Index) full.While {
 
 pub fn fullPtrType(tree: Ast, node: Node.Index) ?full.PtrType {
     return switch (tree.nodes.items(.tag)[node]) {
-        .ptr_type_aligned => tree.ptrTypeAligned(node),
+        .ptr_type_aligned => ptrTypeAligned(tree, node),
-        .ptr_type_sentinel => tree.ptrTypeSentinel(node),
+        .ptr_type_sentinel => ptrTypeSentinel(tree, node),
-        .ptr_type => tree.ptrType(node),
+        .ptr_type => ptrTypeSimple(tree, node),
-        .ptr_type_bit_range => tree.ptrTypeBitRange(node),
+        .ptr_type_bit_range => ptrTypeBitRange(tree, node),
         else => null,
     };
 }
 
 pub fn fullIf(tree: Ast, node: Node.Index) ?full.If {
     return switch (tree.nodes.items(.tag)[node]) {
-        .if_simple => tree.ifSimple(node),
+        .if_simple => ifSimple(tree, node),
-        .@"if" => tree.ifFull(node),
+        .@"if" => ifFull(tree, node),
         else => null,
     };
 }
 
 pub fn fullWhile(tree: Ast, node: Node.Index) ?full.While {
     return switch (tree.nodes.items(.tag)[node]) {
-        .while_simple => tree.whileSimple(node),
+        .while_simple => whileSimple(tree, node),
-        .while_cont => tree.whileCont(node),
+        .while_cont => whileCont(tree, node),
-        .@"while" => tree.whileFull(node),
+        .@"while" => whileFull(tree, node),
-        .for_simple => tree.forSimple(node),
+        .for_simple => forSimple(tree, node),
-        .@"for" => tree.forFull(node),
+        .@"for" => forFull(tree, node),
         else => null,
     };
 }
@@ -543,7 +543,9 @@ pub fn lastToken(tree: Ast, node: Ast.Node.Index) Ast.TokenIndex {
                 n = tree.extra_data[cases.end - 1]; // last case
             }
         },
-        .container_decl_arg => {
+        .container_decl_arg,
+        .container_decl_arg_trailing,
+         => {
             const members = tree.extraData(datas[n].rhs, Node.SubRange);
             if (members.end - members.start == 0) {
                 end_offset += 3; // for the rparen + lbrace + rbrace
@@ -567,7 +569,6 @@ pub fn lastToken(tree: Ast, node: Ast.Node.Index) Ast.TokenIndex {
         },
         .array_init_comma,
         .struct_init_comma,
-        .container_decl_arg_trailing,
         .switch_comma,
         => {
             if (datas[n].rhs != 0) {

src/semantic_tokens.zig

@@ -74,9 +74,7 @@ const Builder = struct {
         const starts = tree.tokens.items(.start);
         const next_start = starts[token];
 
-        if (next_start < self.previous_position) {
+        if (next_start < self.previous_position) return;
-            return error.MovedBackwards;
-        }
 
         if (self.previous_token) |prev| {
             // Highlight gaps between AST nodes. These can contain comments or malformed code.
@@ -182,6 +180,8 @@ const Builder = struct {
     }
 
     fn addDirect(self: *Builder, tok_type: TokenType, tok_mod: TokenModifiers, start: usize, length: usize) !void {
+        if (start < self.previous_position) return;
+
         const text = self.handle.tree.source[self.previous_position..start];
         const delta = offsets.indexToPosition(text, text.len, self.encoding);
 
@@ -261,13 +261,8 @@ fn colorIdentifierBasedOnType(builder: *Builder, type_node: analysis.TypeWithHan
     }
 }
 
-const WriteTokensError = error{
-    OutOfMemory,
-    MovedBackwards,
-};
-
 /// HACK self-hosted has not implemented async yet
-fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) WriteTokensError!void {
+fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) error{OutOfMemory}!void {
     if (zig_builtin.zig_backend == .other or zig_builtin.zig_backend == .stage1) {
         const FrameSize = @sizeOf(@Frame(writeNodeTokens));
         var child_frame = try allocator.alignedAlloc(u8, std.Target.stack_align, FrameSize);
@@ -280,7 +275,7 @@ fn callWriteNodeTokens(allocator: std.mem.Allocator, args: anytype) WriteTokensE
     }
 }
 
-fn writeNodeTokens(builder: *Builder, maybe_node: ?Ast.Node.Index) WriteTokensError!void {
+fn writeNodeTokens(builder: *Builder, maybe_node: ?Ast.Node.Index) error{OutOfMemory}!void {
     const node = maybe_node orelse return;
 
     const handle = builder.handle;
@@ -1010,10 +1005,7 @@ pub fn writeAllSemanticTokens(
 
     // reverse the ast from the root declarations
     for (handle.tree.rootDecls()) |child| {
-        writeNodeTokens(&builder, child) catch |err| switch (err) {
+        try writeNodeTokens(&builder, child);
-            error.MovedBackwards => break,
-            else => |e| return e,
-        };
     }
     try builder.finish();
     return builder.toOwnedSlice();

tests/lsp_features/semantic_tokens.zig

@@ -32,6 +32,18 @@ test "semantic tokens - comments" {
     // TODO more tests
 }
 
+test "semantic tokens - string literals" {
+    // https://github.com/zigtools/zls/issues/921
+    try testSemanticTokens(
+        \\"
+        \\"",// 
+        \\"": 
+    ,
+        // no idea if this output is correct but at least it doesn't crash
+        &.{ 1, 3, 3, 8, 0, 1, 0, 2, 4, 0, 0, 0, 2, 9, 0 },
+    );
+}
+
 const file_uri = switch (builtin.os.tag) {
     .windows => "file:///C:/test.zig",
     else => "file:///test.zig",