andr3/zls
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reworked position context to use tokenizer, works in more contexts

Browse Source
This commit is contained in:
Alexandros Naskos 2020-05-27 01:45:18 +03:00
parent 3c0ee21337
commit f6df35e069
2 changed files with 202 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
src/analysis.zig
View File

@ -433,37 +433,48 @@ pub fn collectImports(import_arr: *std.ArrayList([]const u8), tree: *ast.Tree) !
pub fn getFieldAccessTypeNode(
analysis_ctx: *AnalysisContext,
tokenizer: *std.zig.Tokenizer,
line_length: usize,
) ?*ast.Node {
var current_node = analysis_ctx.in_container;
var current_container = analysis_ctx.in_container;
while (true) {
var next = tokenizer.next();
switch (next.id) {
const tok = tokenizer.next();
switch (tok.id) {
.Eof => return current_node,
.Identifier => {
if (getChildOfSlice(analysis_ctx.tree(), analysis_ctx.scope_nodes, tokenizer.buffer[next.loc.start..next.loc.end])) |child| {
if (getChildOfSlice(analysis_ctx.tree(), analysis_ctx.scope_nodes, tokenizer.buffer[tok.loc.start..tok.loc.end])) |child| {
if (resolveTypeOfNode(analysis_ctx, child)) |node_type| {
current_node = node_type;
} else return null;
} else return null;
},
.Period => {
var after_period = tokenizer.next();
if (after_period.id == .Eof or after_period.id == .Comma) {
return current_node;
} else if (after_period.id == .Identifier) {
const after_period = tokenizer.next();
switch (after_period.id) {
.Eof => return current_node,
.Identifier => {
// TODO: This works for now, maybe we should filter based on the partial identifier ourselves?
if (after_period.loc.end == line_length) return current_node;
if (after_period.loc.end == tokenizer.buffer.len) return current_node;
if (getChild(analysis_ctx.tree(), current_node, tokenizer.buffer[after_period.loc.start..after_period.loc.end])) |child| {
if (resolveTypeOfNode(analysis_ctx, child)) |child_type| {
current_node = child_type;
} else return null;
} else return null;
},
.QuestionMark => {
std.debug.warn("TODO: .? support\n", .{});
return null;
},
else => {
std.debug.warn("Unrecognized token {} after period.\n", .{after_period.id});
return null;
},
}
},
.PeriodAsterisk => {
std.debug.warn("TODO: .* support\n", .{});
return null;
},
.LParen => {
switch (current_node.id) {
.FnProto => {
@ -472,7 +483,7 @@ pub fn getFieldAccessTypeNode(
current_node = ret;
// Skip to the right paren
var paren_count: usize = 1;
next = tokenizer.next();
var next = tokenizer.next();
while (next.id != .Eof) : (next = tokenizer.next()) {
if (next.id == .RParen) {
paren_count -= 1;
@ -480,25 +491,21 @@ pub fn getFieldAccessTypeNode(
} else if (next.id == .LParen) {
paren_count += 1;
}
} else unreachable;
} else return null;
} else {
return null;
}
},
else => {},
}
},
.Keyword_const, .Keyword_var => {
next = tokenizer.next();
if (next.id == .Identifier) {
next = tokenizer.next();
if (next.id != .Equal) return null;
continue;
}
.LBracket => {
std.debug.warn("TODO: Implement [] support\n", .{});
return null;
},
else => std.debug.warn("Not implemented; {}\n", .{next.id}),
else => {
std.debug.warn("Unimplemented token: {}\n", .{tok.id});
return null;
}
}
if (current_node.id == .ContainerDecl or current_node.id == .Root) {
current_container = current_node;
}
@ -763,3 +770,147 @@ pub fn getImportStr(tree: *ast.Tree, source_index: usize) ?[]const u8 {
}
return null;
}
const types = @import("types.zig");
pub const SourceRange = std.zig.Token.Loc;
const PositionContext = union(enum) {
builtin: SourceRange,
comment,
string_literal: SourceRange,
field_access: SourceRange,
var_access: SourceRange,
enum_literal,
other,
empty,
// @TODO remove pub
pub fn range(self: PositionContext) ?SourceRange {
return switch (self) {
.builtin => |r| r,
.comment => null,
.string_literal => |r| r,
.field_access => |r| r,
.var_access => |r| r,
.enum_literal => null,
.other => null,
.empty => null,
};
}
};
const StackState = struct {
ctx: PositionContext,
stack_id: enum { Paren, Bracket, Global },
};
fn peek(arr: *std.ArrayList(StackState)) !*StackState {
if (arr.items.len == 0) {
try arr.append(.{ .ctx = .empty, .stack_id = .Global });
}
return &arr.items[arr.items.len - 1];
}
fn tokenRange(token: std.zig.Token) SourceRange {
return .{
.start = token.loc.start,
.end = token.loc.end,
};
}
fn tokenRangeAppend(prev: SourceRange, token: std.zig.Token) SourceRange {
return .{
.start = prev.start,
.end = token.loc.end,
};
}
// @TODO Test this thoroughly
pub fn documentPositionContext(allocator: *std.mem.Allocator, document: types.TextDocument, position: types.Position) !PositionContext {
const line = try document.getLine(@intCast(usize, position.line));
const pos_char = @intCast(usize, position.character) + 1;
const idx = if (pos_char > line.len) line.len else pos_char;
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
var tokenizer = std.zig.Tokenizer.init(line[0..idx]);
var stack = try std.ArrayList(StackState).initCapacity(&arena.allocator, 8);
// @TODO What happens when we don't close a string? (probably an .Eof)
while (true) {
const tok = tokenizer.next();
// Early exits.
switch (tok.id) {
.Invalid, .Invalid_ampersands => {
// Single '@' do not return a builtin token so we check this on our own.
if (line[idx - 1] == '@') {
return PositionContext{
.builtin = .{
.start = idx - 1,
.end = idx,
},
};
}
return .other;
},
.LineComment, .DocComment, .ContainerDocComment => return .comment,
.Eof => break,
else => {},
}
// State changes
var curr_ctx = try peek(&stack);
switch (tok.id) {
.StringLiteral, .MultilineStringLiteralLine => curr_ctx.ctx = .{ .string_literal = tokenRange(tok) },
.Identifier => switch (curr_ctx.ctx) {
.empty => curr_ctx.ctx = .{ .var_access = tokenRange(tok) },
else => {},
},
.Builtin => switch (curr_ctx.ctx) {
.empty => curr_ctx.ctx = .{ .builtin = tokenRange(tok) },
else => {},
},
.Period, .PeriodAsterisk => switch (curr_ctx.ctx) {
.empty => curr_ctx.ctx = .enum_literal,
.enum_literal => curr_ctx.ctx = .empty,
.field_access => {},
.other => {},
else => curr_ctx.ctx = .{
.field_access = tokenRangeAppend(curr_ctx.ctx.range().?, tok),
},
},
.QuestionMark => switch (curr_ctx.ctx) {
.field_access => {},
else => curr_ctx.ctx = .empty,
},
.LParen => try stack.append(.{ .ctx = .empty, .stack_id = .Paren }),
.LBracket => try stack.append(.{ .ctx = .empty, .stack_id = .Bracket }),
.RParen => {
_ = stack.pop();
if (curr_ctx.stack_id != .Paren) {
(try peek(&stack)).ctx = .empty;
}
},
.RBracket => {
_ = stack.pop();
if (curr_ctx.stack_id != .Bracket) {
(try peek(&stack)).ctx = .empty;
}
},
else => curr_ctx.ctx = .empty,
}
switch (curr_ctx.ctx) {
.field_access => |r| curr_ctx.ctx = .{
.field_access = tokenRangeAppend(r, tok),
},
else => {},
}
}
return block: {
if (stack.popOrNull()) |state| break :block state.ctx;
break :block .empty;
};
}

173
src/main.zig
View File

@ -389,7 +389,7 @@ fn hoverDefinitionGlobal(id: i64, pos_index: usize, handle: *DocumentStore.Handl
fn getSymbolFieldAccess(
analysis_ctx: *DocumentStore.AnalysisContext,
position: types.Position,
line_start_idx: usize,
range: analysis.SourceRange,
config: Config,
) !?*std.zig.ast.Node {
const pos_index = try analysis_ctx.handle.document.positionToIndex(position);
@ -397,12 +397,10 @@ fn getSymbolFieldAccess(
if (name.len == 0) return null;
const line = try analysis_ctx.handle.document.getLine(@intCast(usize, position.line));
var tokenizer = std.zig.Tokenizer.init(line[line_start_idx..]);
var tokenizer = std.zig.Tokenizer.init(line[range.start..range.end]);
const line_length = @ptrToInt(name.ptr) - @ptrToInt(line.ptr) + name.len - line_start_idx;
name = try std.mem.dupe(&analysis_ctx.arena.allocator, u8, name);
if (analysis.getFieldAccessTypeNode(analysis_ctx, &tokenizer, line_length)) |container| {
if (analysis.getFieldAccessTypeNode(analysis_ctx, &tokenizer)) |container| {
return analysis.getChild(analysis_ctx.tree(), container, name);
}
return null;
@ -412,14 +410,14 @@ fn gotoDefinitionFieldAccess(
id: i64,
handle: *DocumentStore.Handle,
position: types.Position,
line_start_idx: usize,
range: analysis.SourceRange,
config: Config,
) !void {
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
var analysis_ctx = try document_store.analysisContext(handle, &arena, try handle.document.positionToIndex(position), config.zig_lib_path);
const decl = (try getSymbolFieldAccess(&analysis_ctx, position, line_start_idx, config)) orelse return try respondGeneric(id, null_result_response);
const decl = (try getSymbolFieldAccess(&analysis_ctx, position, range, config)) orelse return try respondGeneric(id, null_result_response);
return try gotoDefinitionSymbol(id, &analysis_ctx, decl);
}
@ -427,14 +425,14 @@ fn hoverDefinitionFieldAccess(
id: i64,
handle: *DocumentStore.Handle,
position: types.Position,
line_start_idx: usize,
range: analysis.SourceRange,
config: Config,
) !void {
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
var analysis_ctx = try document_store.analysisContext(handle, &arena, try handle.document.positionToIndex(position), config.zig_lib_path);
const decl = (try getSymbolFieldAccess(&analysis_ctx, position, line_start_idx, config)) orelse return try respondGeneric(id, null_result_response);
const decl = (try getSymbolFieldAccess(&analysis_ctx, position, range, config)) orelse return try respondGeneric(id, null_result_response);
return try hoverSymbol(id, &analysis_ctx, decl);
}
@ -490,7 +488,7 @@ fn completeGlobal(id: i64, pos_index: usize, handle: *DocumentStore.Handle, conf
});
}
fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.Position, line_start_idx: usize, config: Config) !void {
fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.Position, range: analysis.SourceRange, config: Config) !void {
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
@ -498,10 +496,9 @@ fn completeFieldAccess(id: i64, handle: *DocumentStore.Handle, position: types.P
var completions = std.ArrayList(types.CompletionItem).init(&arena.allocator);
const line = try handle.document.getLine(@intCast(usize, position.line));
var tokenizer = std.zig.Tokenizer.init(line[line_start_idx..]);
const line_length = line.len - line_start_idx;
var tokenizer = std.zig.Tokenizer.init(line[range.start..range.end]);
if (analysis.getFieldAccessTypeNode(&analysis_ctx, &tokenizer, line_length)) |node| {
if (analysis.getFieldAccessTypeNode(&analysis_ctx, &tokenizer)) |node| {
try nodeToCompletion(&completions, &analysis_ctx, handle, node, config);
}
try send(types.Response{
@ -550,136 +547,6 @@ const builtin_completions = block: {
};
};
const PositionContext = union(enum) {
builtin,
comment,
string_literal,
field_access: usize,
var_access,
other,
empty,
};
const token_separators = [_]u8{
' ', '\t', '(', ')', '[', ']',
'{', '}', '|', '=', '!', ';',
',', '?', ':', '%', '+', '*',
'>', '<', '~', '-', '/', '&',
};
fn documentPositionContext(doc: types.TextDocument, pos_index: usize) PositionContext {
// First extract the whole current line up to the cursor.
var curr_position = pos_index;
while (curr_position > 0) : (curr_position -= 1) {
if (doc.text[curr_position - 1] == '\n') break;
}
var line = doc.text[curr_position .. pos_index + 1];
// Strip any leading whitespace.
var skipped_ws: usize = 0;
while (skipped_ws < line.len and (line[skipped_ws] == ' ' or line[skipped_ws] == '\t')) : (skipped_ws += 1) {}
if (skipped_ws >= line.len) return .empty;
line = line[skipped_ws..];
// Quick exit for comment lines and multi line string literals.
if (line.len >= 2 and line[0] == '/' and line[1] == '/')
return .comment;
if (line.len >= 2 and line[0] == '\\' and line[1] == '\\')
return .string_literal;
// TODO: This does not detect if we are in a string literal over multiple lines.
// Find out what context we are in.
// Go over the current line character by character
// and determine the context.
curr_position = 0;
var expr_start: usize = skipped_ws;
// std.debug.warn("{}", .{curr_position});
if (pos_index != 0 and doc.text[pos_index - 1] == ')')
return .{ .field_access = expr_start };
var new_token = true;
var context: PositionContext = .other;
var string_pop_ctx: PositionContext = .other;
while (curr_position < line.len) : (curr_position += 1) {
const c = line[curr_position];
const next_char = if (curr_position < line.len - 1) line[curr_position + 1] else null;
if (context != .string_literal and c == '"') {
expr_start = curr_position + skipped_ws;
context = .string_literal;
continue;
}
if (context == .string_literal) {
// Skip over escaped quotes
if (c == '\\' and next_char != null and next_char.? == '"') {
curr_position += 1;
} else if (c == '"') {
context = string_pop_ctx;
string_pop_ctx = .other;
new_token = true;
}
continue;
}
if (c == '/' and next_char != null and next_char.? == '/') {
context = .comment;
break;
}
if (std.mem.indexOfScalar(u8, &token_separators, c) != null) {
expr_start = curr_position + skipped_ws + 1;
new_token = true;
context = .other;
continue;
}
if (c == '.' and (!new_token or context == .string_literal)) {
new_token = true;
if (next_char != null and next_char.? == '.') continue;
context = .{ .field_access = expr_start };
continue;
}
if (new_token) {
const access_ctx: PositionContext = if (context == .field_access)
.{ .field_access = expr_start }
else
.var_access;
new_token = false;
if (c == '_' or std.ascii.isAlpha(c)) {
context = access_ctx;
} else if (c == '@') {
// This checks for @"..." identifiers by controlling
// the context the string will set after it is over.
if (next_char != null and next_char.? == '"') {
string_pop_ctx = access_ctx;
}
context = .builtin;
} else {
context = .other;
}
continue;
}
if (context == .field_access or context == .var_access or context == .builtin) {
if (c != '_' and !std.ascii.isAlNum(c)) {
context = .other;
}
continue;
}
context = .other;
}
return context;
}
fn loadConfig(folder_path: []const u8) ?Config {
var folder = std.fs.cwd().openDir(folder_path, .{}) catch return null;
defer folder.close();
@ -851,7 +718,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
};
if (pos.character >= 0) {
const pos_index = try handle.document.positionToIndex(pos);
const pos_context = documentPositionContext(handle.document, pos_index);
const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos);
const this_config = configFromUriOr(uri, config);
switch (pos_context) {
@ -865,7 +732,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
},
}),
.var_access, .empty => try completeGlobal(id, pos_index, handle, this_config),
.field_access => |start_idx| try completeFieldAccess(id, handle, pos, start_idx, this_config),
.field_access => |range| try completeFieldAccess(id, handle, pos, range, this_config),
else => try respondGeneric(id, no_completions_response),
}
} else {
@ -894,7 +761,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
};
if (pos.character >= 0) {
const pos_index = try handle.document.positionToIndex(pos);
const pos_context = documentPositionContext(handle.document, pos_index);
const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos);
switch (pos_context) {
.var_access => try gotoDefinitionGlobal(
@ -903,16 +770,18 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
handle,
configFromUriOr(uri, config),
),
.field_access => |start_idx| try gotoDefinitionFieldAccess(
.field_access => |range| try gotoDefinitionFieldAccess(
id,
handle,
pos,
start_idx,
range,
configFromUriOr(uri, config),
),
.string_literal => try gotoDefinitionString(id, pos_index, handle, config),
else => try respondGeneric(id, null_result_response),
}
} else {
try respondGeneric(id, null_result_response);
}
} else if (std.mem.eql(u8, method, "textDocument/hover")) {
const document = params.getValue("textDocument").?.Object;
@ -930,7 +799,7 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
};
if (pos.character >= 0) {
const pos_index = try handle.document.positionToIndex(pos);
const pos_context = documentPositionContext(handle.document, pos_index);
const pos_context = try analysis.documentPositionContext(allocator, handle.document, pos);
switch (pos_context) {
.var_access => try hoverDefinitionGlobal(
@ -939,15 +808,17 @@ fn processJsonRpc(parser: *std.json.Parser, json: []const u8, config: Config) !v
handle,
configFromUriOr(uri, config),
),
.field_access => |start_idx| try hoverDefinitionFieldAccess(
.field_access => |range| try hoverDefinitionFieldAccess(
id,
handle,
pos,
start_idx,
range,
configFromUriOr(uri, config),
),
else => try respondGeneric(id, null_result_response),
}
} else {
try respondGeneric(id, null_result_response);
}
} else if (root.Object.getValue("id")) |_| {
std.debug.warn("Method with return value not implemented: {}", .{method});