Reworked position context to use tokenizer, works in more contexts

src/analysis.zig

@@ -433,37 +433,48 @@ pub fn collectImports(import_arr: *std.ArrayList([]const u8), tree: *ast.Tree) !
 pub fn getFieldAccessTypeNode(
     analysis_ctx: *AnalysisContext,
     tokenizer: *std.zig.Tokenizer,
-    line_length: usize,
 ) ?*ast.Node {
     var current_node = analysis_ctx.in_container;
     var current_container = analysis_ctx.in_container;
 
     while (true) {
-        var next = tokenizer.next();
+        const tok = tokenizer.next();
-        switch (next.id) {
+        switch (tok.id) {
             .Eof => return current_node,
             .Identifier => {
-                if (getChildOfSlice(analysis_ctx.tree(), analysis_ctx.scope_nodes, tokenizer.buffer[next.loc.start..next.loc.end])) |child| {
+                if (getChildOfSlice(analysis_ctx.tree(), analysis_ctx.scope_nodes, tokenizer.buffer[tok.loc.start..tok.loc.end])) |child| {
                     if (resolveTypeOfNode(analysis_ctx, child)) |node_type| {
                         current_node = node_type;
                     } else return null;
                 } else return null;
             },
             .Period => {
-                var after_period = tokenizer.next();
+                const after_period = tokenizer.next();
-                if (after_period.id == .Eof or after_period.id == .Comma) {
+                switch (after_period.id) {
-                    return current_node;
+                    .Eof => return current_node,
-                } else if (after_period.id == .Identifier) {
+                    .Identifier => {
-                    // TODO: This works for now, maybe we should filter based on the partial identifier ourselves?
+                        // TODO: This works for now, maybe we should filter based on the partial identifier ourselves?
-                    if (after_period.loc.end == line_length) return current_node;
+                        if (after_period.loc.end == tokenizer.buffer.len) return current_node;
-
+                        if (getChild(analysis_ctx.tree(), current_node, tokenizer.buffer[after_period.loc.start..after_period.loc.end])) |child| {
-                    if (getChild(analysis_ctx.tree(), current_node, tokenizer.buffer[after_period.loc.start..after_period.loc.end])) |child| {
+                            if (resolveTypeOfNode(analysis_ctx, child)) |child_type| {
-                        if (resolveTypeOfNode(analysis_ctx, child)) |child_type| {
+                                current_node = child_type;
-                            current_node = child_type;
+                            } else return null;
                         } else return null;
-                    } else return null;
+                    },
+                    .QuestionMark => {
+                        std.debug.warn("TODO: .? support\n", .{});
+                        return null;
+                    },
+                    else => {
+                        std.debug.warn("Unrecognized token {} after period.\n", .{after_period.id});
+                        return null;
+                    },
                 }
             },
+            .PeriodAsterisk => {
+                std.debug.warn("TODO: .* support\n", .{});
+                return null;
+            },
             .LParen => {
                 switch (current_node.id) {
                     .FnProto => {
@@ -472,7 +483,7 @@ pub fn getFieldAccessTypeNode(
                             current_node = ret;
                             // Skip to the right paren
                             var paren_count: usize = 1;
-                            next = tokenizer.next();
+                            var next = tokenizer.next();
                             while (next.id != .Eof) : (next = tokenizer.next()) {
                                 if (next.id == .RParen) {
                                     paren_count -= 1;
@@ -480,25 +491,21 @@ pub fn getFieldAccessTypeNode(
                                 } else if (next.id == .LParen) {
                                     paren_count += 1;
                                 }
-                            } else return null;
+                            } else unreachable;
-                        } else {
+                        } else return null;
-                            return null;
-                        }
                     },
                     else => {},
                 }
             },
-            .Keyword_const, .Keyword_var => {
+            .LBracket => {
-                next = tokenizer.next();
+                std.debug.warn("TODO: Implement [] support\n", .{});
-                if (next.id == .Identifier) {
+                return null;
-                    next = tokenizer.next();
-                    if (next.id != .Equal) return null;
-                    continue;
-                }
             },
-            else => std.debug.warn("Not implemented; {}\n", .{next.id}),
+            else => {
+                std.debug.warn("Unimplemented token: {}\n", .{tok.id});
+                return null;
+            }
         }
-
         if (current_node.id == .ContainerDecl or current_node.id == .Root) {
             current_container = current_node;
         }
@@ -763,3 +770,147 @@ pub fn getImportStr(tree: *ast.Tree, source_index: usize) ?[]const u8 {
     }
     return null;
 }
+
+const types = @import("types.zig");
+pub const SourceRange = std.zig.Token.Loc;
+
+const PositionContext = union(enum) {
+    builtin: SourceRange,
+    comment,
+    string_literal: SourceRange,
+    field_access: SourceRange,
+    var_access: SourceRange,
+    enum_literal,
+    other,
+    empty,
+
+    // @TODO remove pub
+    pub fn range(self: PositionContext) ?SourceRange {
+        return switch (self) {
+            .builtin => |r| r,
+            .comment => null,
+            .string_literal => |r| r,
+            .field_access => |r| r,
+            .var_access => |r| r,
+            .enum_literal => null,
+            .other => null,
+            .empty => null,
+        };
+    }
+};
+
+const StackState = struct {
+    ctx: PositionContext,
+    stack_id: enum { Paren, Bracket, Global },
+};
+
+fn peek(arr: *std.ArrayList(StackState)) !*StackState {
+    if (arr.items.len == 0) {
+        try arr.append(.{ .ctx = .empty, .stack_id = .Global });
+    }
+    return &arr.items[arr.items.len - 1];
+}
+
+fn tokenRange(token: std.zig.Token) SourceRange {
+    return .{
+        .start = token.loc.start,
+        .end = token.loc.end,
+    };
+}
+
+fn tokenRangeAppend(prev: SourceRange, token: std.zig.Token) SourceRange {
+    return .{
+        .start = prev.start,
+        .end = token.loc.end,
+    };
+}
+
+// @TODO Test this thoroughly
+pub fn documentPositionContext(allocator: *std.mem.Allocator, document: types.TextDocument, position: types.Position) !PositionContext {
+    const line = try document.getLine(@intCast(usize, position.line));
+    const pos_char = @intCast(usize, position.character) + 1;
+    const idx = if (pos_char > line.len) line.len else pos_char;
+
+    var arena = std.heap.ArenaAllocator.init(allocator);
+    defer arena.deinit();
+
+    var tokenizer = std.zig.Tokenizer.init(line[0..idx]);
+    var stack = try std.ArrayList(StackState).initCapacity(&arena.allocator, 8);
+
+    // @TODO What happens when we don't close a string? (probably an .Eof)
+    while (true) {
+        const tok = tokenizer.next();
+        // Early exits.
+        switch (tok.id) {
+            .Invalid, .Invalid_ampersands => {
+                // Single '@' do not return a builtin token so we check this on our own.
+                if (line[idx - 1] == '@') {
+                    return PositionContext{
+                        .builtin = .{
+                            .start = idx - 1,
+                            .end = idx,
+                        },
+                    };
+                }
+                return .other;
+            },
+            .LineComment, .DocComment, .ContainerDocComment => return .comment,
+            .Eof => break,
+            else => {},
+        }
+
+        // State changes
+        var curr_ctx = try peek(&stack);
+        switch (tok.id) {
+            .StringLiteral, .MultilineStringLiteralLine => curr_ctx.ctx = .{ .string_literal = tokenRange(tok) },
+            .Identifier => switch (curr_ctx.ctx) {
+                .empty => curr_ctx.ctx = .{ .var_access = tokenRange(tok) },
+                else => {},
+            },
+            .Builtin => switch (curr_ctx.ctx) {
+                .empty => curr_ctx.ctx = .{ .builtin = tokenRange(tok) },
+                else => {},
+            },
+            .Period, .PeriodAsterisk => switch (curr_ctx.ctx) {
+                .empty => curr_ctx.ctx = .enum_literal,
+                .enum_literal => curr_ctx.ctx = .empty,
+                .field_access => {},
+                .other => {},
+                else => curr_ctx.ctx = .{
+                    .field_access = tokenRangeAppend(curr_ctx.ctx.range().?, tok),
+                },
+            },
+            .QuestionMark => switch (curr_ctx.ctx) {
+                .field_access => {},
+                else => curr_ctx.ctx = .empty,
+            },
+            .LParen => try stack.append(.{ .ctx = .empty, .stack_id = .Paren }),
+            .LBracket => try stack.append(.{ .ctx = .empty, .stack_id = .Bracket }),
+            .RParen => {
+                _ = stack.pop();
+                if (curr_ctx.stack_id != .Paren) {
+                    (try peek(&stack)).ctx = .empty;
+                }
+            },
+            .RBracket => {
+                _ = stack.pop();
+                if (curr_ctx.stack_id != .Bracket) {
+                    (try peek(&stack)).ctx = .empty;
+                }
+            },
+            else => curr_ctx.ctx = .empty,
+        }
+
+        switch (curr_ctx.ctx) {
+            .field_access => |r| curr_ctx.ctx = .{
+                .field_access = tokenRangeAppend(r, tok),
+            },
+            else => {},
+        }
+    }
+
+    return block: {
+        if (stack.popOrNull()) |state| break :block state.ctx;
+        break :block .empty;
+    };
+}

src/main.zig

@@ -389,7 +389,7 @@ fn hoverDefinitionGlobal(id: i64, pos_index: usize, handle: *DocumentStore.Handl
 fn getSymbolFieldAccess(
     analysis_ctx: *DocumentStore.AnalysisContext,
     position: types.Position,
-    line_start_idx: usize,
+    range: analysis.SourceRange,
     config: Config,
 ) !?*std.zig.ast.Node {
     const pos_index = try analysis_ctx.handle.document.positionToIndex(position);
@@ -397,12 +397,10 @@ fn getSymbolFieldAccess(
     if (name.len == 0) return null;
 
     const line = try analysis_ctx.handle.document.getLine(@intCast(usize, position.line));
-    var tokenizer = std.zig.Tokenizer.init(line[line_start_idx..]);
+    var tokenizer = std.zig.Tokenizer.init(line[range.start..range.end]);
 
-    const line_length = @ptrToInt(name.ptr) - @ptrToInt(line.ptr) + name.len - line_start_idx;
     name = try std.mem.dupe(&analysis_ctx.arena.allocator, u8, name);
-
+    if (analysis.getFieldAccessTypeNode(analysis_ctx, &tokenizer)) |container| {
-    if (analysis.getFieldAccessTypeNode(analysis_ctx, &tokenizer, line_length)) |container| {
         return analysis.getChild(analysis_ctx.tree(), container, name);
     }
     return null;
@@ -412,14 +410,14 @@ fn gotoDefinitionFieldAccess(
     id: i64,
     handle: *DocumentStore.Handle,
     position: types.Position,
-    line_start_idx: usize,
+    range: analysis.SourceRange,
     config: Config,
 ) !void {
     var arena = std.heap.ArenaAllocator.init(allocator);
     defer arena.deinit();
 
     var analysis_ctx = try document_store.analysisContext(handle, &arena, try handle.document.positionToIndex(position), config.zig_lib_path);
-    const decl = (try getSymbolFieldAccess(&analysis_ctx, position, line_start_idx, config)) orelse return try respondGeneric(id, null_result_response);
+    const decl = (try getSymbolFieldAccess(&analysis_ctx, position, range, config)) orelse return try respondGeneric(id, null_result_response);
     return try gotoDefinitionSymbol(id, &analysis_ctx, decl);
 }
 
@@ -427,14 +425,14 @@ fn hoverDefinitionFieldAccess(
     id: i64,
     handle: *DocumentStore.Handle,
     position: types.Position,
-    line_start_idx: usize,
+    range: analysis.SourceRange,
     config: Config,
 ) !void {
     var arena = std.heap.ArenaAllocator.init(allocator);
     defer arena.deinit();
 
     var analysis_ctx = try document_store.analysisContext(handle, &arena, try handle.document.positionToIndex(position), config.zig_lib_path);
-    const decl = (try getSymbolFieldAccess(&analysis_ctx, position, line_start_idx, config)) orelse return try respondGeneric(id, null_result_response);
+    const decl = (try getSymbolFieldAccess(&analysis_ctx, position, range, config)) orelse return try respondGeneric(id, null_result_response);
     return try hoverSymbol(id, &analysis_ctx, decl);
 }
 
@@ -490,7 +488,7 @@ fn completeGlobal(id: i64, pos_index: usize, handle: *DocumentStore.Handle, conf
     });
 }
 
-fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.Position, line_start_idx: usize, config: Config) !void {
+fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.Position, range: analysis.SourceRange, config: Config) !void {
     var arena = std.heap.ArenaAllocator.init(allocator);
     defer arena.deinit();
 
@@ -498,10 +496,9 @@ fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.P
     var completions = std.ArrayList(types.CompletionItem).init(&arena.allocator);
 
     const line = try handle.document.getLine(@intCast(usize, position.line));
-    var tokenizer = std.zig.Tokenizer.init(line[line_start_idx..]);
+    var tokenizer = std.zig.Tokenizer.init(line[range.start..range.end]);
-    const line_length = line.len - line_start_idx;
 
-    if (analysis.getFieldAccessTypeNode(&analysis_ctx, &tokenizer, line_length)) |node| {
+    if (analysis.getFieldAccessTypeNode(&analysis_ctx, &tokenizer)) |node| {
         try nodeToCompletion(&completions, &analysis_ctx, handle, node, config);
     }
     try send(types.Response{
@@ -550,136 +547,6 @@ const builtin_completions = block: {
     };
 };
 
-const PositionContext = union(enum) {
-    builtin,
-    comment,
-    string_literal,
-    field_access: usize,
-    var_access,
-    other,
-    empty,
-};
-
-const token_separators = [_]u8{
-    ' ', '\t', '(', ')', '[', ']',
-    '{', '}',  '|', '=', '!', ';',
-    ',', '?',  ':', '%', '+', '*',
-    '>', '<',  '~', '-', '/', '&',
-};
-
-fn documentPositionContext(doc: types.TextDocument, pos_index: usize) PositionContext {
-    // First extract the whole current line up to the cursor.
-    var curr_position = pos_index;
-    while (curr_position > 0) : (curr_position -= 1) {
-        if (doc.text[curr_position - 1] == '\n') break;
-    }
-
-    var line = doc.text[curr_position .. pos_index + 1];
-    // Strip any leading whitespace.
-    var skipped_ws: usize = 0;
-    while (skipped_ws < line.len and (line[skipped_ws] == ' ' or line[skipped_ws] == '\t')) : (skipped_ws += 1) {}
-    if (skipped_ws >= line.len) return .empty;
-    line = line[skipped_ws..];
-
-    // Quick exit for comment lines and multi line string literals.
-    if (line.len >= 2 and line[0] == '/' and line[1] == '/')
-        return .comment;
-    if (line.len >= 2 and line[0] == '\\' and line[1] == '\\')
-        return .string_literal;
-
-    // TODO: This does not detect if we are in a string literal over multiple lines.
-    // Find out what context we are in.
-    // Go over the current line character by character
-    // and determine the context.
-    curr_position = 0;
-    var expr_start: usize = skipped_ws;
-
-    // std.debug.warn("{}", .{curr_position});
-
-    if (pos_index != 0 and doc.text[pos_index - 1] == ')')
-        return .{ .field_access = expr_start };
-
-    var new_token = true;
-    var context: PositionContext = .other;
-    var string_pop_ctx: PositionContext = .other;
-    while (curr_position < line.len) : (curr_position += 1) {
-        const c = line[curr_position];
-        const next_char = if (curr_position < line.len - 1) line[curr_position + 1] else null;
-
-        if (context != .string_literal and c == '"') {
-            expr_start = curr_position + skipped_ws;
-            context = .string_literal;
-            continue;
-        }
-
-        if (context == .string_literal) {
-            // Skip over escaped quotes
-            if (c == '\\' and next_char != null and next_char.? == '"') {
-                curr_position += 1;
-            } else if (c == '"') {
-                context = string_pop_ctx;
-                string_pop_ctx = .other;
-                new_token = true;
-            }
-
-            continue;
-        }
-
-        if (c == '/' and next_char != null and next_char.? == '/') {
-            context = .comment;
-            break;
-        }
-
-        if (std.mem.indexOfScalar(u8, &token_separators, c) != null) {
-            expr_start = curr_position + skipped_ws + 1;
-            new_token = true;
-            context = .other;
-            continue;
-        }
-
-        if (c == '.' and (!new_token or context == .string_literal)) {
-            new_token = true;
-            if (next_char != null and next_char.? == '.') continue;
-            context = .{ .field_access = expr_start };
-            continue;
-        }
-
-        if (new_token) {
-            const access_ctx: PositionContext = if (context == .field_access)
-                .{ .field_access = expr_start }
-            else
-                .var_access;
-
-            new_token = false;
-
-            if (c == '_' or std.ascii.isAlpha(c)) {
-                context = access_ctx;
-            } else if (c == '@') {
-                // This checks for @"..." identifiers by controlling
-                // the context the string will set after it is over.
-                if (next_char != null and next_char.? == '"') {
-                    string_pop_ctx = access_ctx;
-                }
-                context = .builtin;
-            } else {
-                context = .other;
-            }
-            continue;
-        }
-
-        if (context == .field_access or context == .var_access or context == .builtin) {
-            if (c != '_' and !std.ascii.isAlNum(c)) {
-                context = .other;
-            }
-            continue;
-        }
-
-        context = .other;
-    }
-
-    return context;
-}
-
 fn loadConfig(folder_path: []const u8) ?Config {
     var folder = std.fs.cwd().openDir(folder_path, .{}) catch return null;
     defer folder.close();
@@ -851,7 +718,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
         };
         if (pos.character >= 0) {
             const pos_index = try handle.document.positionToIndex(pos);
-            const pos_context = documentPositionContext(handle.document, pos_index);
+            const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos);
 
             const this_config = configFromUriOr(uri, config);
             switch (pos_context) {
@@ -865,7 +732,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
                     },
                 }),
                 .var_access, .empty => try completeGlobal(id, pos_index, handle, this_config),
-                .field_access => |start_idx| try completeFieldAccess(id, handle, pos, start_idx, this_config),
+                .field_access => |range| try completeFieldAccess(id, handle, pos, range, this_config),
                 else => try respondGeneric(id, no_completions_response),
             }
         } else {
@@ -894,7 +761,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
         };
         if (pos.character >= 0) {
             const pos_index = try handle.document.positionToIndex(pos);
-            const pos_context = documentPositionContext(handle.document, pos_index);
+            const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos);
 
             switch (pos_context) {
                 .var_access => try gotoDefinitionGlobal(
@@ -903,16 +770,18 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
                     handle,
                     configFromUriOr(uri, config),
                 ),
-                .field_access => |start_idx| try gotoDefinitionFieldAccess(
+                .field_access => |range| try gotoDefinitionFieldAccess(
                     id,
                     handle,
                     pos,
-                    start_idx,
+                    range,
                     configFromUriOr(uri, config),
                 ),
                 .string_literal => try gotoDefinitionString(id, pos_index, handle, config),
                 else => try respondGeneric(id, null_result_response),
             }
+        } else {
+            try respondGeneric(id, null_result_response);
         }
     } else if (std.mem.eql(u8, method, "textDocument/hover")) {
         const document = params.getValue("textDocument").?.Object;
@@ -930,7 +799,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
         };
         if (pos.character >= 0) {
             const pos_index = try handle.document.positionToIndex(pos);
-            const pos_context = documentPositionContext(handle.document, pos_index);
+            const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos);
 
             switch (pos_context) {
                 .var_access => try hoverDefinitionGlobal(
@@ -939,15 +808,17 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
                     handle,
                     configFromUriOr(uri, config),
                 ),
-                .field_access => |start_idx| try hoverDefinitionFieldAccess(
+                .field_access => |range| try hoverDefinitionFieldAccess(
                     id,
                     handle,
                     pos,
-                    start_idx,
+                    range,
                     configFromUriOr(uri, config),
                 ),
                 else => try respondGeneric(id, null_result_response),
             }
+        } else {
+            try respondGeneric(id, null_result_response);
         }
     } else if (root.Object.getValue("id")) |_| {
         std.debug.warn("Method with return value not implemented: {}", .{method});