From b623f141ea190762c804a63ad669994b4d468903 Mon Sep 17 00:00:00 2001 From: Techatrix <19954306+Techatrix@users.noreply.github.com> Date: Tue, 30 May 2023 01:59:46 +0200 Subject: [PATCH] revive translateC support --- src/DocumentStore.zig | 20 ++-- src/ZigCompileServer.zig | 168 ++++++++++++++++++++++++++++ src/translate_c.zig | 104 +++++++++++++---- tests/language_features/cimport.zig | 52 ++++++++- 4 files changed, 315 insertions(+), 29 deletions(-) create mode 100644 src/ZigCompileServer.zig diff --git a/src/DocumentStore.zig b/src/DocumentStore.zig index 4de9e40..703fbaa 100644 --- a/src/DocumentStore.zig +++ b/src/DocumentStore.zig @@ -988,7 +988,10 @@ pub fn collectIncludeDirs( const native_include_dirs = try native_paths.include_dirs.toOwnedSlice(); defer allocator.free(native_include_dirs); - include_dirs.appendSliceAssumeCapacity(native_include_dirs); + for (native_include_dirs) |native_include_dir| { + var array = std.ArrayListUnmanaged(u8).fromOwnedSliceSentinel(0, native_include_dir); + include_dirs.appendAssumeCapacity(try array.toOwnedSlice(allocator)); + } for (build_file_includes_paths) |include_path| { const absolute_path = if (std.fs.path.isAbsolute(include_path)) @@ -1015,10 +1018,6 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index if (!std.process.can_spawn) return null; - // FIXME: Re-enable cimport resolution once https://github.com/ziglang/zig/issues/15025 is resolved - // Tracking issue: https://github.com/zigtools/zls/issues/1080 - if (true) return null; - const index = std.mem.indexOfScalar(Ast.Node.Index, handle.cimports.items(.node), node) orelse return null; const hash: Hash = handle.cimports.items(.hash)[index]; @@ -1039,12 +1038,19 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index return null; }; - var result = (try translate_c.translate( + const maybe_result = translate_c.translate( self.allocator, self.config.*, include_dirs.items, source, - )) orelse return null; + ) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => |e| { + log.err("failed to translate cimport: {}", .{e}); + return null; + }, + }; + var result = maybe_result orelse return null; self.cimports.putNoClobber(self.allocator, hash, result) catch result.deinit(self.allocator); diff --git a/src/ZigCompileServer.zig b/src/ZigCompileServer.zig new file mode 100644 index 0000000..749e2e3 --- /dev/null +++ b/src/ZigCompileServer.zig @@ -0,0 +1,168 @@ +//! modified version of https://github.com/ziglang/zig/blob/master/lib/std/zig/Server.zig +//! I don't know why but this code seems to work +//! zig binary serialization library in stdlib, when? + +in: std.fs.File, +out: std.fs.File, +pooler: std.io.Poller(StreamEnum), + +const StreamEnum = enum { in }; + +pub const Options = struct { + gpa: Allocator, + in: std.fs.File, + out: std.fs.File, +}; + +pub fn init(options: Options) Client { + var s: Client = .{ + .in = options.in, + .out = options.out, + .pooler = std.io.poll(options.gpa, StreamEnum, .{ .in = options.in }), + }; + return s; +} + +pub fn deinit(s: *Client) void { + s.pooler.deinit(); + s.* = undefined; +} + +pub fn receiveMessage(client: *Client) !InMessage.Header { + const Header = InMessage.Header; + const fifo = client.pooler.fifo(.in); + + while (try client.pooler.poll()) { + const buf = fifo.readableSlice(0); + assert(fifo.readableLength() == buf.len); + if (buf.len >= @sizeOf(Header)) { + // workaround for https://github.com/ziglang/zig/issues/14904 + const bytes_len = bswap_and_workaround_u32(buf[4..][0..4]); + const tag = bswap_and_workaround_tag(buf[0..][0..4]); + + if (buf.len - @sizeOf(Header) >= bytes_len) { + fifo.discard(@sizeOf(Header)); + return .{ + .tag = tag, + .bytes_len = bytes_len, + }; + } else { + const needed = bytes_len - (buf.len - @sizeOf(Header)); + const write_buffer = try fifo.writableWithSize(needed); + const amt = try client.in.readAll(write_buffer); + fifo.update(amt); + continue; + } + } + + const write_buffer = try fifo.writableWithSize(256); + const amt = try client.in.read(write_buffer); + fifo.update(amt); + } + return error.Timeout; +} + +pub fn receiveEmitBinPath(client: *Client) !InMessage.EmitBinPath { + const reader = client.pooler.fifo(.in).reader(); + return reader.readStruct(InMessage.EmitBinPath); +} + +pub fn receiveErrorBundle(client: *Client) !InMessage.ErrorBundle { + const reader = client.pooler.fifo(.in).reader(); + return .{ + .extra_len = try reader.readIntLittle(u32), + .string_bytes_len = try reader.readIntLittle(u32), + }; +} + +pub fn receiveBytes(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u8 { + const reader = client.pooler.fifo(.in).reader(); + const result = try reader.readAllAlloc(allocator, len); + if (result.len != len) return error.UnexpectedEOF; + return result; +} + +pub fn receiveIntArray(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u32 { + const reader = client.pooler.fifo(.in).reader(); + var array_list = std.ArrayListAligned(u8, 4).init(allocator); + errdefer array_list.deinit(); + try reader.readAllArrayListAligned(4, &array_list, len); + const bytes = try array_list.toOwnedSlice(); + const result = std.mem.bytesAsSlice(u32, bytes); + if (need_bswap) { + bswap_u32_array(result); + } + return result; +} + +pub fn serveMessage( + client: *const Client, + header: OutMessage.Header, + bufs: []const []const u8, +) !void { + var iovecs: [10]std.os.iovec_const = undefined; + const header_le = bswap(header); + iovecs[0] = .{ + .iov_base = @ptrCast([*]const u8, &header_le), + .iov_len = @sizeOf(OutMessage.Header), + }; + for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { + iovec.* = .{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + } + try client.out.writevAll(iovecs[0 .. bufs.len + 1]); +} + +fn bswap(x: anytype) @TypeOf(x) { + if (!need_bswap) return x; + + const T = @TypeOf(x); + switch (@typeInfo(T)) { + .Enum => return @intToEnum(T, @byteSwap(@enumToInt(x))), + .Int => return @byteSwap(x), + .Struct => |info| switch (info.layout) { + .Extern => { + var result: T = undefined; + inline for (info.fields) |field| { + @field(result, field.name) = bswap(@field(x, field.name)); + } + return result; + }, + .Packed => { + const I = info.backing_integer.?; + return @bitCast(T, @byteSwap(@bitCast(I, x))); + }, + .Auto => @compileError("auto layout struct"), + }, + else => @compileError("bswap on type " ++ @typeName(T)), + } +} + +fn bswap_u32_array(slice: []u32) void { + comptime assert(need_bswap); + for (slice) |*elem| elem.* = @byteSwap(elem.*); +} + +/// workaround for https://github.com/ziglang/zig/issues/14904 +fn bswap_and_workaround_u32(bytes_ptr: *const [4]u8) u32 { + return std.mem.readIntLittle(u32, bytes_ptr); +} + +/// workaround for https://github.com/ziglang/zig/issues/14904 +fn bswap_and_workaround_tag(bytes_ptr: *const [4]u8) InMessage.Tag { + const int = std.mem.readIntLittle(u32, bytes_ptr); + return @intToEnum(InMessage.Tag, int); +} + +const OutMessage = std.zig.Client.Message; +const InMessage = std.zig.Server.Message; + +const Client = @This(); +const builtin = @import("builtin"); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const native_endian = builtin.target.cpu.arch.endian(); +const need_bswap = native_endian != .Little; diff --git a/src/translate_c.zig b/src/translate_c.zig index 97c0182..45fe112 100644 --- a/src/translate_c.zig +++ b/src/translate_c.zig @@ -6,6 +6,7 @@ const ast = @import("ast.zig"); const tracy = @import("tracy.zig"); const Ast = std.zig.Ast; const URI = @import("uri.zig"); +const ZCS = @import("ZigCompileServer.zig"); const log = std.log.scoped(.zls_translate_c); /// converts a `@cInclude` node into an equivalent c header file @@ -133,7 +134,12 @@ pub const Result = union(enum) { /// returns a URI to the generated zig file on success or the content of stderr on failure /// null indicates a failure which is automatically logged /// Caller owns returned memory. -pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []const []const u8, source: []const u8) error{OutOfMemory}!?Result { +pub fn translate( + allocator: std.mem.Allocator, + config: Config, + include_dirs: []const []const u8, + source: []const u8, +) !?Result { const tracy_zone = tracy.trace(@src()); defer tracy_zone.end(); @@ -157,12 +163,12 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c const base_args = &[_][]const u8{ config.zig_exe_path orelse return null, "translate-c", - "--enable-cache", "--zig-lib-dir", config.zig_lib_path orelse return null, "--cache-dir", config.global_cache_path.?, "-lc", + "--listen=-", }; const argc = base_args.len + 2 * include_dirs.len + 1; @@ -178,28 +184,86 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c argv.appendAssumeCapacity(file_path); - const result = std.ChildProcess.exec(.{ - .allocator = allocator, - .argv = argv.items, - }) catch |err| { - log.err("Failed to execute zig translate-c process, error: {}", .{err}); + var process = std.ChildProcess.init(argv.items, allocator); + process.stdin_behavior = .Pipe; + process.stdout_behavior = .Pipe; + process.stderr_behavior = .Pipe; + + errdefer |err| if (!zig_builtin.is_test) blk: { + const joined = std.mem.join(allocator, " ", argv.items) catch break :blk; + defer allocator.free(joined); + if (process.stderr) |stderr| { + const stderr_output = stderr.readToEndAlloc(allocator, std.math.maxInt(usize)) catch break :blk; + defer allocator.free(stderr_output); + log.err("failed zig translate-c command:\n{s}\nstderr:{s}\nerror:{}\n", .{ joined, stderr_output, err }); + } else { + log.err("failed zig translate-c command:\n{s}\nerror:{}\n", .{ joined, err }); + } + }; + + process.spawn() catch |err| { + log.err("failed to spawn zig translate-c process, error: {}", .{err}); return null; }; - defer allocator.free(result.stdout); - defer allocator.free(result.stderr); - - return switch (result.term) { - .Exited => |code| if (code == 0) { - return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result.stdout, '\n')) }; - } else { - return Result{ .failure = try allocator.dupe(u8, std.mem.sliceTo(result.stderr, '\n')) }; - }, - else => { - log.err("zig translate-c process terminated '{}'", .{result.term}); - return null; - }, + defer _ = process.wait() catch |wait_err| blk: { + log.err("zig translate-c process did not terminate, error: {}", .{wait_err}); + break :blk process.kill() catch |kill_err| { + std.debug.panic("failed to terminate zig translate-c process, error: {}", .{kill_err}); + }; }; + + var zcs = ZCS.init(.{ + .gpa = allocator, + .in = process.stdout.?, + .out = process.stdin.?, + }); + defer zcs.deinit(); + + try zcs.serveMessage(.{ .tag = .update, .bytes_len = 0 }, &.{}); + try zcs.serveMessage(.{ .tag = .exit, .bytes_len = 0 }, &.{}); + + while (true) { + const header = try zcs.receiveMessage(); + // log.debug("received header: {}", .{header}); + + switch (header.tag) { + .zig_version => { + // log.debug("zig-version: {s}", .{zcs.receive_fifo.readableSliceOfLen(header.bytes_len)}); + zcs.pooler.fifo(.in).discard(header.bytes_len); + }, + .emit_bin_path => { + const body_size = @sizeOf(std.zig.Server.Message.EmitBinPath); + if (header.bytes_len <= body_size) return error.InvalidResponse; + + const trailing_size = header.bytes_len - body_size; + + _ = try zcs.receiveEmitBinPath(); + + const result_path = try zcs.receiveBytes(allocator, trailing_size); + defer allocator.free(result_path); + + return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result_path, '\n')) }; + }, + .error_bundle => { + const error_bundle_header = try zcs.receiveErrorBundle(); + + const extra = try zcs.receiveIntArray(allocator, error_bundle_header.extra_len); + defer allocator.free(extra); + + const string_bytes = try zcs.receiveBytes(allocator, error_bundle_header.string_bytes_len); + defer allocator.free(string_bytes); + + const error_bundle = std.zig.ErrorBundle{ .string_bytes = string_bytes, .extra = extra }; + + return Result{ .failure = try allocator.dupe(u8, error_bundle.getCompileLogOutput()) }; + }, + else => { + log.warn("received unexpected message {} from zig compile server", .{header.tag}); + return null; + }, + } + } } fn extractString(str: []const u8) []const u8 { diff --git a/tests/language_features/cimport.zig b/tests/language_features/cimport.zig index a0f4d9c..31ce0cb 100644 --- a/tests/language_features/cimport.zig +++ b/tests/language_features/cimport.zig @@ -8,6 +8,24 @@ const translate_c = zls.translate_c; const allocator: std.mem.Allocator = std.testing.allocator; +test "zig compile server - translate c" { + var result1 = try testTranslate( + \\void foo(int); + \\void bar(float*); + ); + defer result1.deinit(allocator); + try std.testing.expect(result1 == .success); + + // TODO the zig compiler doesn't seem to report error bundles for translate-c + // Hopefully I can fix that once llvm-16 finished compiling :) + + var result2 = testTranslate( + \\#include + ); + defer if (result2) |*r| r.deinit(allocator) else |_| {}; + try std.testing.expectError(error.Timeout, result2); +} + test "convertCInclude - empty" { try testConvertCInclude("@cImport()", ""); try testConvertCInclude("@cImport({})", ""); @@ -29,7 +47,7 @@ test "convertCInclude - cInclude" { ); } -test "translate_c - cDefine" { +test "convertCInclude - cDefine" { try testConvertCInclude( \\@cImport(@cDefine("FOO", "BAR")) , @@ -42,7 +60,7 @@ test "translate_c - cDefine" { ); } -test "translate_c - cUndef" { +test "convertCInclude - cUndef" { try testConvertCInclude( \\@cImport(@cUndef("FOO")) , @@ -85,3 +103,33 @@ fn testConvertCInclude(cimport_source: []const u8, expected: []const u8) !void { try std.testing.expectEqualStrings(expected, trimmed_output); } + +fn testTranslate(c_source: []const u8) !translate_c.Result { + if (!std.process.can_spawn) return error.SkipZigTest; + + var config: zls.Config = .{}; + defer std.json.parseFree(zls.Config, allocator, config); + + var runtime_zig_version: ?zls.ZigVersionWrapper = null; + defer if (runtime_zig_version) |*v| v.free(); + + try zls.configuration.configChanged(&config, &runtime_zig_version, allocator, null); + + if (config.global_cache_path == null or + config.zig_exe_path == null or + config.zig_lib_path == null) return error.SkipZigTest; + + const result = (try translate_c.translate(allocator, config, &.{}, c_source)).?; + + switch (result) { + .success => |uri| { + const path = try zls.URI.parse(allocator, uri); + defer allocator.free(path); + try std.testing.expect(std.fs.path.isAbsolute(path)); + }, + .failure => |message| { + try std.testing.expect(message.len != 0); + }, + } + return result; +}