Add a replay feature to zls (#857)

* add config options for `zls --replay`

* implement `zls --replay`

* remove carriage return from zls replay files

* add missing arguments for Server.init in tests

.gitignore

@@ -2,3 +2,4 @@
 zig-*
 debug
 release
+*.zlsreplay

README.md

@@ -86,6 +86,9 @@ The following options are currently available.
 | `include_at_in_builtins` | `bool` | `false` | Whether the @ sign should be part of the completion of builtins |
 | `skip_std_references` | `bool` | `false` | When true, skips searching for references in std. Improves lookup speed for functions in user's code. Renaming and go-to-definition will continue to work as is |
 | `max_detail_length` | `usize` | `1048576` | The detail field of completions is truncated to be no longer than this (in bytes) |
+| `record_session` | `bool` | `false` | When true, zls will record all request is receives and write in into `record_session_path`, so that they can replayed with `zls replay` |
+| `record_session_path` | `?[]const u8` | `null` | Output file path when `record_session` is set. The recommended file extension *.zlsreplay |
+| `replay_session_path` | `?[]const u8` | `null` | Used when calling `zls replay` for specifying the replay file. If no extra argument is given `record_session_path` is used as the default path. |
 | `builtin_path` | `?[]const u8` | `null` | Path to 'builtin;' useful for debugging, automatically set if let null |
 | `zig_lib_path` | `?[]const u8` | `null` | Zig library path, e.g. `/path/to/zig/lib/zig`, used to analyze std library imports |
 | `zig_exe_path` | `?[]const u8` | `null` | Zig executable path, e.g. `/path/to/zig/zig`, used to run the custom build runner. If `null`, zig is looked up in `PATH`. Will be used to infer the zig standard library path if none is provided |

schema.json

@@ -89,6 +89,21 @@
             "type": "integer",
             "default": "1048576"
         },
+        "record_session": {
+            "description": "When true, zls will record all request is receives and write in into `record_session_path`, so that they can replayed with `zls replay`",
+            "type": "boolean",
+            "default": "false"
+        },
+        "record_session_path": {
+            "description": "Output file path when `record_session` is set. The recommended file extension *.zlsreplay",
+            "type": "string",
+            "default": "null"
+        },
+        "replay_session_path": {
+            "description": "Used when calling `zls replay` for specifying the replay file. If no extra argument is given `record_session_path` is used as the default path.",
+            "type": "string",
+            "default": "null"
+        },
         "builtin_path": {
             "description": "Path to 'builtin;' useful for debugging, automatically set if let null",
             "type": "string",

src/Config.zig

@@ -55,6 +55,15 @@ skip_std_references: bool = false,
 /// The detail field of completions is truncated to be no longer than this (in bytes)
 max_detail_length: usize = 1048576,
 
+/// When true, zls will record all request is receives and write in into `record_session_path`, so that they can replayed with `zls replay`
+record_session: bool = false,
+
+/// Output file path when `record_session` is set. The recommended file extension *.zlsreplay
+record_session_path: ?[]const u8 = null,
+
+/// Used when calling `zls replay` for specifying the replay file. If no extra argument is given `record_session_path` is used as the default path.
+replay_session_path: ?[]const u8 = null,
+
 /// Path to 'builtin;' useful for debugging, automatically set if let null
 builtin_path: ?[]const u8 = null,
 

src/Header.zig

@@ -12,7 +12,7 @@ pub fn deinit(self: @This(), allocator: std.mem.Allocator) void {
 }
 
 // Caller owns returned memory.
-pub fn parse(allocator: std.mem.Allocator, reader: anytype) !Header {
+pub fn parse(allocator: std.mem.Allocator, include_carriage_return: bool, reader: anytype) !Header {
     var r = Header{
         .content_length = undefined,
         .content_type = null,
@@ -23,11 +23,15 @@ pub fn parse(allocator: std.mem.Allocator, reader: anytype) !Header {
     while (true) {
         const header = try reader.readUntilDelimiterAlloc(allocator, '\n', 0x100);
         defer allocator.free(header);
+        if (include_carriage_return) {
             if (header.len == 0 or header[header.len - 1] != '\r') return error.MissingCarriageReturn;
             if (header.len == 1) break;
+        } else {
+            if (header.len == 0) break;
+        }
 
         const header_name = header[0 .. std.mem.indexOf(u8, header, ": ") orelse return error.MissingColon];
-        const header_value = header[header_name.len + 2 .. header.len - 1];
+        const header_value = header[header_name.len + 2 .. header.len - @boolToInt(include_carriage_return)];
         if (std.mem.eql(u8, header_name, "Content-Length")) {
             if (header_value.len == 0) return error.MissingHeaderValue;
             r.content_length = std.fmt.parseInt(usize, header_value, 10) catch return error.InvalidContentLength;
@@ -43,17 +47,11 @@ pub fn parse(allocator: std.mem.Allocator, reader: anytype) !Header {
     return r;
 }
 
-pub fn format(
+pub fn write(header: Header, include_carriage_return: bool, writer: anytype) @TypeOf(writer).Error!void {
-    header: Header,
+    const seperator: []const u8 = if (include_carriage_return) "\r\n" else "\n";
-    comptime unused_fmt_string: []const u8,
+    try writer.print("Content-Length: {}{s}", .{header.content_length, seperator});
-    options: std.fmt.FormatOptions,
-    writer: anytype,
-) @TypeOf(writer).Error!void {
-    _ = options;
-    std.debug.assert(unused_fmt_string.len == 0);
-    try writer.print("Content-Length: {}\r\n", .{header.content_length});
     if (header.content_type) |content_type| {
-        try writer.print("Content-Type: {s}\r\n", .{content_type});
+        try writer.print("Content-Type: {s}{s}", .{content_type, seperator});
     }
-    try writer.writeAll("\r\n");
+    try writer.writeAll(seperator);
 }

src/Server.zig

@@ -37,6 +37,8 @@ document_store: DocumentStore = undefined,
 builtin_completions: std.ArrayListUnmanaged(types.CompletionItem),
 client_capabilities: ClientCapabilities = .{},
 outgoing_messages: std.ArrayListUnmanaged([]const u8) = .{},
+recording_enabled: bool,
+replay_enabled: bool,
 offset_encoding: offsets.Encoding = .@"utf-16",
 status: enum {
     /// the server has not received a `initialize` request
@@ -176,7 +178,13 @@ fn sendInternal(
     try server.outgoing_messages.append(server.allocator, message);
 }
 
-fn showMessage(server: *Server, message_type: types.MessageType, message: []const u8) void {
+fn showMessage(
+    server: *Server,
+    message_type: types.MessageType,
+    comptime fmt: []const u8,
+    args: anytype,
+) void {
+    const message = std.fmt.allocPrint(server.arena.allocator(), fmt, args) catch return;
     server.sendNotification("window/showMessage", types.ShowMessageParams{
         .type = message_type,
         .message = message,
@@ -1661,19 +1669,20 @@ fn initializeHandler(server: *Server, request: types.InitializeParams) !types.In
         const zig_exe_version = std.SemanticVersion.parse(env.version) catch break :blk;
 
         if (zig_builtin.zig_version.order(zig_exe_version) == .gt) {
-            const version_mismatch_message = try std.fmt.allocPrint(
+            server.showMessage(.Warning,
-                server.arena.allocator(),
+                \\ZLS was built with Zig {}, but your Zig version is {s}. Update Zig to avoid unexpected behavior.
-                "ZLS was built with Zig {}, but your Zig version is {s}. Update Zig to avoid unexpected behavior.",
+            , .{ zig_builtin.zig_version, env.version });
-                .{ zig_builtin.zig_version, env.version },
-            );
-            server.showMessage(.Warning, version_mismatch_message);
         }
     } else {
-        server.showMessage(
+        server.showMessage(.Warning,
-            .Warning,
             \\ZLS failed to find Zig. Please add Zig to your PATH or set the zig_exe_path config option in your zls.json.
-            ,
+        , .{});
-        );
+    }
+
+    if (server.recording_enabled) {
+        server.showMessage(.Info,
+            \\This zls session is being recorded to {?s}.
+        , .{server.config.record_session_path});
     }
 
     return .{
@@ -1810,6 +1819,11 @@ fn registerCapability(server: *Server, method: []const u8) !void {
 }
 
 fn requestConfiguration(server: *Server) !void {
+    if (server.recording_enabled) {
+        log.info("workspace/configuration are disabled during a recording session!", .{});
+        return;
+    }
+
     const configuration_items = comptime confi: {
         var comp_confi: [std.meta.fields(Config).len]types.ConfigurationItem = undefined;
         inline for (std.meta.fields(Config)) |field, index| {
@@ -1831,6 +1845,10 @@ fn requestConfiguration(server: *Server) !void {
 }
 
 fn handleConfiguration(server: *Server, json: std.json.Value) error{OutOfMemory}!void {
+    if (server.replay_enabled) {
+        log.info("workspace/configuration are disabled during a replay!", .{});
+        return;
+    }
     log.info("Setting configuration...", .{});
 
     // NOTE: Does this work with other editors?
@@ -3028,6 +3046,8 @@ pub fn init(
     allocator: std.mem.Allocator,
     config: *Config,
     config_path: ?[]const u8,
+    recording_enabled: bool,
+    replay_enabled: bool,
 ) !Server {
     // TODO replace global with something like an Analyser struct
     // which contains using_trail & resolve_trail and place it inside Server
@@ -3068,6 +3088,8 @@ pub fn init(
         .allocator = allocator,
         .document_store = document_store,
         .builtin_completions = builtin_completions,
+        .recording_enabled = recording_enabled,
+        .replay_enabled = replay_enabled,
         .status = .uninitialized,
     };
 }

src/config_gen/config.json

@@ -119,6 +119,27 @@
             "default": "1048576",
             "setup_question": null
         },
+        {
+            "name": "record_session",
+            "description": "When true, zls will record all request is receives and write in into `record_session_path`, so that they can replayed with `zls replay`",
+            "type": "bool",
+            "default": "false",
+            "setup_question": null
+        },
+        {
+            "name": "record_session_path",
+            "description": "Output file path when `record_session` is set. The recommended file extension *.zlsreplay",
+            "type": "?[]const u8",
+            "default": "null",
+            "setup_question": null
+        },
+        {
+            "name": "replay_session_path",
+            "description": "Used when calling `zls replay` for specifying the replay file. If no extra argument is given `record_session_path` is used as the default path.",
+            "type": "?[]const u8",
+            "default": "null",
+            "setup_question": null
+        },
         {
             "name": "builtin_path",
             "description": "Path to 'builtin;' useful for debugging, automatically set if let null",

src/main.zig

@@ -35,10 +35,17 @@ pub fn log(
     std.debug.print(format ++ "\n", args);
 }
 
-fn loop(server: *Server) !void {
+fn loop(
-    const reader = std.io.getStdIn().reader();
+    server: *Server,
-
+    record_file: ?std.fs.File,
+    replay_file: ?std.fs.File,
+) !void {
+    const std_in = std.io.getStdIn().reader();
     const std_out = std.io.getStdOut().writer();
+
+    var buffered_reader = std.io.bufferedReader(if (replay_file) |file| file.reader() else std_in);
+    const reader = buffered_reader.reader();
+
     var buffered_writer = std.io.bufferedWriter(std_out);
     const writer = buffered_writer.writer();
 
@@ -49,24 +56,96 @@ fn loop(server: *Server) !void {
         // write server -> client messages
         for (server.outgoing_messages.items) |outgoing_message| {
             const header = Header{ .content_length = outgoing_message.len };
-            try writer.print("{}{s}", .{ header, outgoing_message });
+            try header.write(true, writer);
-            try buffered_writer.flush();
+            try writer.writeAll(outgoing_message);
         }
+        try buffered_writer.flush();
         for (server.outgoing_messages.items) |outgoing_message| {
             server.allocator.free(outgoing_message);
         }
         server.outgoing_messages.clearRetainingCapacity();
 
         // read and handle client -> server message
-        const header = try Header.parse(arena.allocator(), reader);
+        const header = try Header.parse(arena.allocator(), replay_file == null, reader);
 
         const json_message = try arena.allocator().alloc(u8, header.content_length);
         try reader.readNoEof(json_message);
 
+        if (record_file) |file| {
+            try header.write(false, file.writer());
+            try file.writeAll(json_message);
+        }
+
         server.processJsonRpc(&arena, json_message);
     }
 }
 
+fn getRecordFile(config: Config) ?std.fs.File {
+    if (!config.record_session) return null;
+
+    if (config.record_session_path) |record_path| {
+        if (std.fs.createFileAbsolute(record_path, .{})) |file| {
+            std.debug.print("recording to {s}\n", .{record_path});
+            return file;
+        } else |err| {
+            std.log.err("failed to create record file at {s}: {}", .{ record_path, err });
+            return null;
+        }
+    } else {
+        std.log.err("`record_session` is set but `record_session_path` is unspecified", .{});
+        return null;
+    }
+}
+
+fn getReplayFile(config: Config) ?std.fs.File {
+    const replay_path = config.replay_session_path orelse config.record_session_path orelse return null;
+
+    if (std.fs.openFileAbsolute(replay_path, .{})) |file| {
+        std.debug.print("replaying from {s}\n", .{replay_path});
+        return file;
+    } else |err| {
+        std.log.err("failed to open replay file at {s}: {}", .{ replay_path, err });
+        return null;
+    }
+}
+
+/// when recording we add a message that saves the current configuration in the replay
+/// when replaying we read this message and replace the current config
+fn updateConfig(
+    allocator: std.mem.Allocator,
+    config: *Config,
+    record_file: ?std.fs.File,
+    replay_file: ?std.fs.File,
+) !void {
+    if (record_file) |file| {
+        var cfg = config.*;
+        cfg.record_session = false;
+        cfg.record_session_path = null;
+        cfg.replay_session_path = null;
+
+        var buffer = std.ArrayListUnmanaged(u8){};
+        defer buffer.deinit(allocator);
+
+        try std.json.stringify(cfg, .{}, buffer.writer(allocator));
+        const header = Header{ .content_length = buffer.items.len };
+        try header.write(false, file.writer());
+        try file.writeAll(buffer.items);
+    }
+
+    if (replay_file) |file| {
+        const header = try Header.parse(allocator, false, file.reader());
+        defer header.deinit(allocator);
+        const json_message = try allocator.alloc(u8, header.content_length);
+        defer allocator.free(json_message);
+        try file.reader().readNoEof(json_message);
+
+        var token_stream = std.json.TokenStream.init(json_message);
+        const new_config = try std.json.parse(Config, &token_stream, .{ .allocator = allocator });
+        std.json.parseFree(Config, config.*, .{ .allocator = allocator });
+        config.* = new_config;
+    }
+}
+
 const ConfigWithPath = struct {
     config: Config,
     config_path: ?[]const u8,
@@ -75,44 +154,28 @@ const ConfigWithPath = struct {
 fn getConfig(
     allocator: std.mem.Allocator,
     config_path: ?[]const u8,
-    /// If true, and the provided config_path is non-null, frees
-    /// the aforementioned path, in the case that it is
-    /// not returned.
-    free_old_config_path: bool,
 ) !ConfigWithPath {
     if (config_path) |path| {
-        if (configuration.loadFromFile(allocator, path)) |conf| {
+        if (configuration.loadFromFile(allocator, path)) |config| {
-            return ConfigWithPath{
+            return ConfigWithPath{ .config = config, .config_path = path };
-                .config = conf,
-                .config_path = path,
-            };
         }
         std.debug.print(
             \\Could not open configuration file '{s}'
             \\Falling back to a lookup in the local and global configuration folders
             \\
         , .{path});
-        if (free_old_config_path) {
-            allocator.free(path);
-        }
     }
 
     if (try known_folders.getPath(allocator, .local_configuration)) |path| {
-        if (configuration.loadFromFolder(allocator, path)) |conf| {
+        if (configuration.loadFromFolder(allocator, path)) |config| {
-            return ConfigWithPath{
+            return ConfigWithPath{ .config = config, .config_path = path };
-                .config = conf,
-                .config_path = path,
-            };
         }
         allocator.free(path);
     }
 
     if (try known_folders.getPath(allocator, .global_configuration)) |path| {
-        if (configuration.loadFromFolder(allocator, path)) |conf| {
+        if (configuration.loadFromFolder(allocator, path)) |config| {
-            return ConfigWithPath{
+            return ConfigWithPath{ .config = config, .config_path = path };
-                .config = conf,
-                .config_path = path,
-            };
         }
         allocator.free(path);
     }
@@ -123,22 +186,33 @@ fn getConfig(
     };
 }
 
-const ParseArgsResult = enum { proceed, exit };
+const ParseArgsResult = struct {
-fn parseArgs(
+    action: enum { proceed, exit },
-    allocator: std.mem.Allocator,
+    config_path: ?[]const u8,
-    config: *ConfigWithPath,
+    replay_enabled: bool,
-) !ParseArgsResult {
+    replay_session_path: ?[]const u8,
+};
+
+fn parseArgs(allocator: std.mem.Allocator) !ParseArgsResult {
+    var result = ParseArgsResult{
+        .action = .exit,
+        .config_path = null,
+        .replay_enabled = false,
+        .replay_session_path = null,
+    };
+
     const ArgId = enum {
         help,
         version,
         config,
+        replay,
         @"enable-debug-log",
         @"show-config-path",
         @"config-path",
     };
     const arg_id_map = std.ComptimeStringMap(ArgId, comptime blk: {
         const fields = @typeInfo(ArgId).Enum.fields;
-        const KV = std.meta.Tuple(&.{ []const u8, ArgId });
+        const KV = struct { []const u8, ArgId };
         var pairs: [fields.len]KV = undefined;
         for (pairs) |*pair, i| pair.* = .{ fields[i].name, @intToEnum(ArgId, fields[i].value) };
         break :blk pairs[0..];
@@ -155,10 +229,11 @@ fn parseArgs(
         var cmd_infos: InfoMap = InfoMap.init(.{
             .help = "Prints this message.",
             .version = "Prints the compiler version with which the server was compiled.",
+            .config = "Run the ZLS configuration wizard.",
+            .replay = "Replay a previous recorded zls session",
             .@"enable-debug-log" = "Enables debug logs.",
             .@"config-path" = "Specify the path to a configuration file specifying LSP behaviour.",
             .@"show-config-path" = "Prints the path to the configuration file to stdout",
-            .config = "Run the ZLS configuration wizard.",
         });
         var info_it = cmd_infos.iterator();
         while (info_it.next()) |entry| {
@@ -174,9 +249,6 @@ fn parseArgs(
 
     // Makes behavior of enabling debug more logging consistent regardless of argument order.
     var specified = std.enums.EnumArray(ArgId, bool).initFill(false);
-    var config_path: ?[]const u8 = null;
-    errdefer if (config_path) |path| allocator.free(path);
-
     const stdout = std.io.getStdOut().writer();
     const stderr = std.io.getStdErr().writer();
 
@@ -184,60 +256,61 @@ fn parseArgs(
         if (!std.mem.startsWith(u8, tok, "--") or tok.len == 2) {
             try stderr.print("{s}\n", .{help_message});
             try stderr.print("Unexpected positional argument '{s}'.\n", .{tok});
-            return .exit;
+            return result;
         }
 
         const argname = tok["--".len..];
         const id = arg_id_map.get(argname) orelse {
             try stderr.print("{s}\n", .{help_message});
             try stderr.print("Unrecognized argument '{s}'.\n", .{argname});
-            return .exit;
+            return result;
         };
 
         if (specified.get(id)) {
             try stderr.print("{s}\n", .{help_message});
             try stderr.print("Duplicate argument '{s}'.\n", .{argname});
-            return .exit;
+            return result;
         }
         specified.set(id, true);
 
         switch (id) {
-            .help => {},
+            .help, .version, .@"enable-debug-log", .config, .@"show-config-path" => {},
-            .version => {},
-            .@"enable-debug-log" => {},
-            .config => {},
-            .@"show-config-path" => {},
             .@"config-path" => {
                 const path = args_it.next() orelse {
                     try stderr.print("Expected configuration file path after --config-path argument.\n", .{});
-                    return .exit;
+                    return result;
                 };
-                config.config_path = try allocator.dupe(u8, path);
+                result.config_path = try allocator.dupe(u8, path);
+            },
+            .replay => {
+                result.replay_enabled = true;
+                const path = args_it.next() orelse break;
+                result.replay_session_path = try allocator.dupe(u8, path);
             },
         }
     }
 
     if (specified.get(.help)) {
         try stderr.print("{s}\n", .{help_message});
-        return .exit;
+        return result;
     }
     if (specified.get(.version)) {
-        try std.io.getStdOut().writeAll(build_options.version ++ "\n");
+        try stdout.writeAll(build_options.version ++ "\n");
-        return .exit;
+        return result;
     }
     if (specified.get(.config)) {
         try setup.wizard(allocator);
-        return .exit;
+        return result;
     }
     if (specified.get(.@"enable-debug-log")) {
         actual_log_level = .debug;
         logger.info("Enabled debug logging.\n", .{});
     }
     if (specified.get(.@"config-path")) {
-        std.debug.assert(config.config_path != null);
+        std.debug.assert(result.config_path != null);
     }
     if (specified.get(.@"show-config-path")) {
-        const new_config = try getConfig(allocator, config.config_path, true);
+        const new_config = try getConfig(allocator, result.config_path);
         defer if (new_config.config_path) |path| allocator.free(path);
         defer std.json.parseFree(Config, new_config.config, .{ .allocator = allocator });
 
@@ -250,10 +323,11 @@ fn parseArgs(
         } else {
             logger.err("Failed to find zls.json!\n", .{});
         }
-        return .exit;
+        return result;
     }
 
-    return .proceed;
+    result.action = .proceed;
+    return result;
 }
 
 const stack_frames = switch (zig_builtin.mode) {
@@ -271,30 +345,45 @@ pub fn main() !void {
     var failing_allocator_state = if(build_options.enable_failing_allocator) debug.FailingAllocator.init(inner_allocator, build_options.enable_failing_allocator_likelihood) else void{};
     const allocator: std.mem.Allocator = if(build_options.enable_failing_allocator) failing_allocator_state.allocator() else inner_allocator;
 
-    var config = ConfigWithPath{
+    const result = try parseArgs(allocator);
-        .config = undefined,
+    defer if (result.config_path) |path| allocator.free(path);
-        .config_path = null,
+    defer if (result.replay_session_path) |path| allocator.free(path);
-    };
+    switch (result.action) {
-    defer if (config.config_path) |path| allocator.free(path);
-
-    switch (try parseArgs(allocator, &config)) {
         .proceed => {},
         .exit => return,
     }
 
-    config = try getConfig(allocator, config.config_path, true);
+    var config = try getConfig(allocator, result.config_path);
     defer std.json.parseFree(Config, config.config, .{ .allocator = allocator });
+    defer if (config.config_path) |path| allocator.free(path);
+
+    if (result.replay_enabled and config.config.replay_session_path == null and config.config.record_session_path == null) {
+        logger.err("No replay file specified", .{});
+        return;
+    }
 
     if (config.config_path == null) {
         logger.info("No config file zls.json found.", .{});
     }
 
+    const record_file = if (!result.replay_enabled) getRecordFile(config.config) else null;
+    defer if (record_file) |file| file.close();
+
+    const replay_file = if (result.replay_enabled) getReplayFile(config.config) else null;
+    defer if (replay_file) |file| file.close();
+
+    std.debug.assert(record_file == null or replay_file == null);
+
+    try updateConfig(allocator, &config.config, record_file, replay_file);
+
     var server = try Server.init(
         allocator,
         &config.config,
         config.config_path,
+        record_file != null,
+        replay_file != null,
     );
     defer server.deinit();
 
-    try loop(&server);
+    try loop(&server, record_file, replay_file);
 }

tests/context.zig

@@ -40,7 +40,7 @@ pub const Context = struct {
 
         config.* = default_config;
 
-        var server = try Server.init(allocator, config, null);
+        var server = try Server.init(allocator, config, null, false, false);
         errdefer server.deinit();
 
         var context: Context = .{