From f6df35e069d151b7e42f6dcd62a1b125107657ab Mon Sep 17 00:00:00 2001 From: Alexandros Naskos Date: Wed, 27 May 2020 01:45:18 +0300 Subject: [PATCH] Reworked position context to use tokenizer, works in more contexts --- src/analysis.zig | 209 ++++++++++++++++++++++++++++++++++++++++------- src/main.zig | 173 +++++---------------------------------- 2 files changed, 202 insertions(+), 180 deletions(-) diff --git a/src/analysis.zig b/src/analysis.zig index 1b76d39..3badfdf 100644 --- a/src/analysis.zig +++ b/src/analysis.zig @@ -433,37 +433,48 @@ pub fn collectImports(import_arr: *std.ArrayList([]const u8), tree: *ast.Tree) ! pub fn getFieldAccessTypeNode( analysis_ctx: *AnalysisContext, tokenizer: *std.zig.Tokenizer, - line_length: usize, ) ?*ast.Node { var current_node = analysis_ctx.in_container; var current_container = analysis_ctx.in_container; while (true) { - var next = tokenizer.next(); - switch (next.id) { + const tok = tokenizer.next(); + switch (tok.id) { .Eof => return current_node, .Identifier => { - if (getChildOfSlice(analysis_ctx.tree(), analysis_ctx.scope_nodes, tokenizer.buffer[next.loc.start..next.loc.end])) |child| { + if (getChildOfSlice(analysis_ctx.tree(), analysis_ctx.scope_nodes, tokenizer.buffer[tok.loc.start..tok.loc.end])) |child| { if (resolveTypeOfNode(analysis_ctx, child)) |node_type| { current_node = node_type; } else return null; } else return null; }, .Period => { - var after_period = tokenizer.next(); - if (after_period.id == .Eof or after_period.id == .Comma) { - return current_node; - } else if (after_period.id == .Identifier) { - // TODO: This works for now, maybe we should filter based on the partial identifier ourselves? - if (after_period.loc.end == line_length) return current_node; - - if (getChild(analysis_ctx.tree(), current_node, tokenizer.buffer[after_period.loc.start..after_period.loc.end])) |child| { - if (resolveTypeOfNode(analysis_ctx, child)) |child_type| { - current_node = child_type; + const after_period = tokenizer.next(); + switch (after_period.id) { + .Eof => return current_node, + .Identifier => { + // TODO: This works for now, maybe we should filter based on the partial identifier ourselves? + if (after_period.loc.end == tokenizer.buffer.len) return current_node; + if (getChild(analysis_ctx.tree(), current_node, tokenizer.buffer[after_period.loc.start..after_period.loc.end])) |child| { + if (resolveTypeOfNode(analysis_ctx, child)) |child_type| { + current_node = child_type; + } else return null; } else return null; - } else return null; + }, + .QuestionMark => { + std.debug.warn("TODO: .? support\n", .{}); + return null; + }, + else => { + std.debug.warn("Unrecognized token {} after period.\n", .{after_period.id}); + return null; + }, } }, + .PeriodAsterisk => { + std.debug.warn("TODO: .* support\n", .{}); + return null; + }, .LParen => { switch (current_node.id) { .FnProto => { @@ -472,7 +483,7 @@ pub fn getFieldAccessTypeNode( current_node = ret; // Skip to the right paren var paren_count: usize = 1; - next = tokenizer.next(); + var next = tokenizer.next(); while (next.id != .Eof) : (next = tokenizer.next()) { if (next.id == .RParen) { paren_count -= 1; @@ -480,25 +491,21 @@ pub fn getFieldAccessTypeNode( } else if (next.id == .LParen) { paren_count += 1; } - } else return null; - } else { - return null; - } + } else unreachable; + } else return null; }, else => {}, } }, - .Keyword_const, .Keyword_var => { - next = tokenizer.next(); - if (next.id == .Identifier) { - next = tokenizer.next(); - if (next.id != .Equal) return null; - continue; - } + .LBracket => { + std.debug.warn("TODO: Implement [] support\n", .{}); + return null; }, - else => std.debug.warn("Not implemented; {}\n", .{next.id}), + else => { + std.debug.warn("Unimplemented token: {}\n", .{tok.id}); + return null; + } } - if (current_node.id == .ContainerDecl or current_node.id == .Root) { current_container = current_node; } @@ -763,3 +770,147 @@ pub fn getImportStr(tree: *ast.Tree, source_index: usize) ?[]const u8 { } return null; } + +const types = @import("types.zig"); +pub const SourceRange = std.zig.Token.Loc; + +const PositionContext = union(enum) { + builtin: SourceRange, + comment, + string_literal: SourceRange, + field_access: SourceRange, + var_access: SourceRange, + enum_literal, + other, + empty, + + // @TODO remove pub + pub fn range(self: PositionContext) ?SourceRange { + return switch (self) { + .builtin => |r| r, + .comment => null, + .string_literal => |r| r, + .field_access => |r| r, + .var_access => |r| r, + .enum_literal => null, + .other => null, + .empty => null, + }; + } +}; + +const StackState = struct { + ctx: PositionContext, + stack_id: enum { Paren, Bracket, Global }, +}; + +fn peek(arr: *std.ArrayList(StackState)) !*StackState { + if (arr.items.len == 0) { + try arr.append(.{ .ctx = .empty, .stack_id = .Global }); + } + return &arr.items[arr.items.len - 1]; +} + +fn tokenRange(token: std.zig.Token) SourceRange { + return .{ + .start = token.loc.start, + .end = token.loc.end, + }; +} + +fn tokenRangeAppend(prev: SourceRange, token: std.zig.Token) SourceRange { + return .{ + .start = prev.start, + .end = token.loc.end, + }; +} + +// @TODO Test this thoroughly +pub fn documentPositionContext(allocator: *std.mem.Allocator, document: types.TextDocument, position: types.Position) !PositionContext { + const line = try document.getLine(@intCast(usize, position.line)); + const pos_char = @intCast(usize, position.character) + 1; + const idx = if (pos_char > line.len) line.len else pos_char; + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var tokenizer = std.zig.Tokenizer.init(line[0..idx]); + var stack = try std.ArrayList(StackState).initCapacity(&arena.allocator, 8); + + // @TODO What happens when we don't close a string? (probably an .Eof) + while (true) { + const tok = tokenizer.next(); + // Early exits. + switch (tok.id) { + .Invalid, .Invalid_ampersands => { + // Single '@' do not return a builtin token so we check this on our own. + if (line[idx - 1] == '@') { + return PositionContext{ + .builtin = .{ + .start = idx - 1, + .end = idx, + }, + }; + } + return .other; + }, + .LineComment, .DocComment, .ContainerDocComment => return .comment, + .Eof => break, + else => {}, + } + + // State changes + var curr_ctx = try peek(&stack); + switch (tok.id) { + .StringLiteral, .MultilineStringLiteralLine => curr_ctx.ctx = .{ .string_literal = tokenRange(tok) }, + .Identifier => switch (curr_ctx.ctx) { + .empty => curr_ctx.ctx = .{ .var_access = tokenRange(tok) }, + else => {}, + }, + .Builtin => switch (curr_ctx.ctx) { + .empty => curr_ctx.ctx = .{ .builtin = tokenRange(tok) }, + else => {}, + }, + .Period, .PeriodAsterisk => switch (curr_ctx.ctx) { + .empty => curr_ctx.ctx = .enum_literal, + .enum_literal => curr_ctx.ctx = .empty, + .field_access => {}, + .other => {}, + else => curr_ctx.ctx = .{ + .field_access = tokenRangeAppend(curr_ctx.ctx.range().?, tok), + }, + }, + .QuestionMark => switch (curr_ctx.ctx) { + .field_access => {}, + else => curr_ctx.ctx = .empty, + }, + .LParen => try stack.append(.{ .ctx = .empty, .stack_id = .Paren }), + .LBracket => try stack.append(.{ .ctx = .empty, .stack_id = .Bracket }), + .RParen => { + _ = stack.pop(); + if (curr_ctx.stack_id != .Paren) { + (try peek(&stack)).ctx = .empty; + } + }, + .RBracket => { + _ = stack.pop(); + if (curr_ctx.stack_id != .Bracket) { + (try peek(&stack)).ctx = .empty; + } + }, + else => curr_ctx.ctx = .empty, + } + + switch (curr_ctx.ctx) { + .field_access => |r| curr_ctx.ctx = .{ + .field_access = tokenRangeAppend(r, tok), + }, + else => {}, + } + } + + return block: { + if (stack.popOrNull()) |state| break :block state.ctx; + break :block .empty; + }; +} diff --git a/src/main.zig b/src/main.zig index 9f1cfc4..950340d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -389,7 +389,7 @@ fn hoverDefinitionGlobal(id: i64, pos_index: usize, handle: *DocumentStore.Handl fn getSymbolFieldAccess( analysis_ctx: *DocumentStore.AnalysisContext, position: types.Position, - line_start_idx: usize, + range: analysis.SourceRange, config: Config, ) !?*std.zig.ast.Node { const pos_index = try analysis_ctx.handle.document.positionToIndex(position); @@ -397,12 +397,10 @@ fn getSymbolFieldAccess( if (name.len == 0) return null; const line = try analysis_ctx.handle.document.getLine(@intCast(usize, position.line)); - var tokenizer = std.zig.Tokenizer.init(line[line_start_idx..]); + var tokenizer = std.zig.Tokenizer.init(line[range.start..range.end]); - const line_length = @ptrToInt(name.ptr) - @ptrToInt(line.ptr) + name.len - line_start_idx; name = try std.mem.dupe(&analysis_ctx.arena.allocator, u8, name); - - if (analysis.getFieldAccessTypeNode(analysis_ctx, &tokenizer, line_length)) |container| { + if (analysis.getFieldAccessTypeNode(analysis_ctx, &tokenizer)) |container| { return analysis.getChild(analysis_ctx.tree(), container, name); } return null; @@ -412,14 +410,14 @@ fn gotoDefinitionFieldAccess( id: i64, handle: *DocumentStore.Handle, position: types.Position, - line_start_idx: usize, + range: analysis.SourceRange, config: Config, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var analysis_ctx = try document_store.analysisContext(handle, &arena, try handle.document.positionToIndex(position), config.zig_lib_path); - const decl = (try getSymbolFieldAccess(&analysis_ctx, position, line_start_idx, config)) orelse return try respondGeneric(id, null_result_response); + const decl = (try getSymbolFieldAccess(&analysis_ctx, position, range, config)) orelse return try respondGeneric(id, null_result_response); return try gotoDefinitionSymbol(id, &analysis_ctx, decl); } @@ -427,14 +425,14 @@ fn hoverDefinitionFieldAccess( id: i64, handle: *DocumentStore.Handle, position: types.Position, - line_start_idx: usize, + range: analysis.SourceRange, config: Config, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var analysis_ctx = try document_store.analysisContext(handle, &arena, try handle.document.positionToIndex(position), config.zig_lib_path); - const decl = (try getSymbolFieldAccess(&analysis_ctx, position, line_start_idx, config)) orelse return try respondGeneric(id, null_result_response); + const decl = (try getSymbolFieldAccess(&analysis_ctx, position, range, config)) orelse return try respondGeneric(id, null_result_response); return try hoverSymbol(id, &analysis_ctx, decl); } @@ -490,7 +488,7 @@ fn completeGlobal(id: i64, pos_index: usize, handle: *DocumentStore.Handle, conf }); } -fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.Position, line_start_idx: usize, config: Config) !void { +fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.Position, range: analysis.SourceRange, config: Config) !void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); @@ -498,10 +496,9 @@ fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.P var completions = std.ArrayList(types.CompletionItem).init(&arena.allocator); const line = try handle.document.getLine(@intCast(usize, position.line)); - var tokenizer = std.zig.Tokenizer.init(line[line_start_idx..]); - const line_length = line.len - line_start_idx; + var tokenizer = std.zig.Tokenizer.init(line[range.start..range.end]); - if (analysis.getFieldAccessTypeNode(&analysis_ctx, &tokenizer, line_length)) |node| { + if (analysis.getFieldAccessTypeNode(&analysis_ctx, &tokenizer)) |node| { try nodeToCompletion(&completions, &analysis_ctx, handle, node, config); } try send(types.Response{ @@ -550,136 +547,6 @@ const builtin_completions = block: { }; }; -const PositionContext = union(enum) { - builtin, - comment, - string_literal, - field_access: usize, - var_access, - other, - empty, -}; - -const token_separators = [_]u8{ - ' ', '\t', '(', ')', '[', ']', - '{', '}', '|', '=', '!', ';', - ',', '?', ':', '%', '+', '*', - '>', '<', '~', '-', '/', '&', -}; - -fn documentPositionContext(doc: types.TextDocument, pos_index: usize) PositionContext { - // First extract the whole current line up to the cursor. - var curr_position = pos_index; - while (curr_position > 0) : (curr_position -= 1) { - if (doc.text[curr_position - 1] == '\n') break; - } - - var line = doc.text[curr_position .. pos_index + 1]; - // Strip any leading whitespace. - var skipped_ws: usize = 0; - while (skipped_ws < line.len and (line[skipped_ws] == ' ' or line[skipped_ws] == '\t')) : (skipped_ws += 1) {} - if (skipped_ws >= line.len) return .empty; - line = line[skipped_ws..]; - - // Quick exit for comment lines and multi line string literals. - if (line.len >= 2 and line[0] == '/' and line[1] == '/') - return .comment; - if (line.len >= 2 and line[0] == '\\' and line[1] == '\\') - return .string_literal; - - // TODO: This does not detect if we are in a string literal over multiple lines. - // Find out what context we are in. - // Go over the current line character by character - // and determine the context. - curr_position = 0; - var expr_start: usize = skipped_ws; - - // std.debug.warn("{}", .{curr_position}); - - if (pos_index != 0 and doc.text[pos_index - 1] == ')') - return .{ .field_access = expr_start }; - - var new_token = true; - var context: PositionContext = .other; - var string_pop_ctx: PositionContext = .other; - while (curr_position < line.len) : (curr_position += 1) { - const c = line[curr_position]; - const next_char = if (curr_position < line.len - 1) line[curr_position + 1] else null; - - if (context != .string_literal and c == '"') { - expr_start = curr_position + skipped_ws; - context = .string_literal; - continue; - } - - if (context == .string_literal) { - // Skip over escaped quotes - if (c == '\\' and next_char != null and next_char.? == '"') { - curr_position += 1; - } else if (c == '"') { - context = string_pop_ctx; - string_pop_ctx = .other; - new_token = true; - } - - continue; - } - - if (c == '/' and next_char != null and next_char.? == '/') { - context = .comment; - break; - } - - if (std.mem.indexOfScalar(u8, &token_separators, c) != null) { - expr_start = curr_position + skipped_ws + 1; - new_token = true; - context = .other; - continue; - } - - if (c == '.' and (!new_token or context == .string_literal)) { - new_token = true; - if (next_char != null and next_char.? == '.') continue; - context = .{ .field_access = expr_start }; - continue; - } - - if (new_token) { - const access_ctx: PositionContext = if (context == .field_access) - .{ .field_access = expr_start } - else - .var_access; - - new_token = false; - - if (c == '_' or std.ascii.isAlpha(c)) { - context = access_ctx; - } else if (c == '@') { - // This checks for @"..." identifiers by controlling - // the context the string will set after it is over. - if (next_char != null and next_char.? == '"') { - string_pop_ctx = access_ctx; - } - context = .builtin; - } else { - context = .other; - } - continue; - } - - if (context == .field_access or context == .var_access or context == .builtin) { - if (c != '_' and !std.ascii.isAlNum(c)) { - context = .other; - } - continue; - } - - context = .other; - } - - return context; -} - fn loadConfig(folder_path: []const u8) ?Config { var folder = std.fs.cwd().openDir(folder_path, .{}) catch return null; defer folder.close(); @@ -851,7 +718,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v }; if (pos.character >= 0) { const pos_index = try handle.document.positionToIndex(pos); - const pos_context = documentPositionContext(handle.document, pos_index); + const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos); const this_config = configFromUriOr(uri, config); switch (pos_context) { @@ -865,7 +732,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v }, }), .var_access, .empty => try completeGlobal(id, pos_index, handle, this_config), - .field_access => |start_idx| try completeFieldAccess(id, handle, pos, start_idx, this_config), + .field_access => |range| try completeFieldAccess(id, handle, pos, range, this_config), else => try respondGeneric(id, no_completions_response), } } else { @@ -894,7 +761,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v }; if (pos.character >= 0) { const pos_index = try handle.document.positionToIndex(pos); - const pos_context = documentPositionContext(handle.document, pos_index); + const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos); switch (pos_context) { .var_access => try gotoDefinitionGlobal( @@ -903,16 +770,18 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v handle, configFromUriOr(uri, config), ), - .field_access => |start_idx| try gotoDefinitionFieldAccess( + .field_access => |range| try gotoDefinitionFieldAccess( id, handle, pos, - start_idx, + range, configFromUriOr(uri, config), ), .string_literal => try gotoDefinitionString(id, pos_index, handle, config), else => try respondGeneric(id, null_result_response), } + } else { + try respondGeneric(id, null_result_response); } } else if (std.mem.eql(u8, method, "textDocument/hover")) { const document = params.getValue("textDocument").?.Object; @@ -930,7 +799,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v }; if (pos.character >= 0) { const pos_index = try handle.document.positionToIndex(pos); - const pos_context = documentPositionContext(handle.document, pos_index); + const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos); switch (pos_context) { .var_access => try hoverDefinitionGlobal( @@ -939,15 +808,17 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v handle, configFromUriOr(uri, config), ), - .field_access => |start_idx| try hoverDefinitionFieldAccess( + .field_access => |range| try hoverDefinitionFieldAccess( id, handle, pos, - start_idx, + range, configFromUriOr(uri, config), ), else => try respondGeneric(id, null_result_response), } + } else { + try respondGeneric(id, null_result_response); } } else if (root.Object.getValue("id")) |_| { std.debug.warn("Method with return value not implemented: {}", .{method});