Merge pull request #1136 from Techatrix/revive-translateC

revive C import support

src/DocumentStore.zig

@@ -988,7 +988,10 @@ pub fn collectIncludeDirs(
 
     const native_include_dirs = try native_paths.include_dirs.toOwnedSlice();
     defer allocator.free(native_include_dirs);
-    include_dirs.appendSliceAssumeCapacity(native_include_dirs);
+    for (native_include_dirs) |native_include_dir| {
+        var array = std.ArrayListUnmanaged(u8).fromOwnedSliceSentinel(0, native_include_dir);
+        include_dirs.appendAssumeCapacity(try array.toOwnedSlice(allocator));
+    }
 
     for (build_file_includes_paths) |include_path| {
         const absolute_path = if (std.fs.path.isAbsolute(include_path))
@@ -1015,10 +1018,6 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index
 
     if (!std.process.can_spawn) return null;
 
-    // FIXME: Re-enable cimport resolution once https://github.com/ziglang/zig/issues/15025 is resolved
-    // Tracking issue: https://github.com/zigtools/zls/issues/1080
-    if (true) return null;
-
     const index = std.mem.indexOfScalar(Ast.Node.Index, handle.cimports.items(.node), node) orelse return null;
 
     const hash: Hash = handle.cimports.items(.hash)[index];
@@ -1039,12 +1038,19 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index
             return null;
         };
 
-        var result = (try translate_c.translate(
+        const maybe_result = translate_c.translate(
             self.allocator,
             self.config.*,
             include_dirs.items,
             source,
-        )) orelse return null;
+        ) catch |err| switch (err) {
+            error.OutOfMemory => |e| return e,
+            else => |e| {
+                log.err("failed to translate cimport: {}", .{e});
+                return null;
+            },
+        };
+        var result = maybe_result orelse return null;
 
         self.cimports.putNoClobber(self.allocator, hash, result) catch result.deinit(self.allocator);
 

src/ZigCompileServer.zig

@@ -0,0 +1,168 @@
+//! modified version of https://github.com/ziglang/zig/blob/master/lib/std/zig/Server.zig
+//! I don't know why but this code seems to work
+//! zig binary serialization library in stdlib, when?
+
+in: std.fs.File,
+out: std.fs.File,
+pooler: std.io.Poller(StreamEnum),
+
+const StreamEnum = enum { in };
+
+pub const Options = struct {
+    gpa: Allocator,
+    in: std.fs.File,
+    out: std.fs.File,
+};
+
+pub fn init(options: Options) Client {
+    var s: Client = .{
+        .in = options.in,
+        .out = options.out,
+        .pooler = std.io.poll(options.gpa, StreamEnum, .{ .in = options.in }),
+    };
+    return s;
+}
+
+pub fn deinit(s: *Client) void {
+    s.pooler.deinit();
+    s.* = undefined;
+}
+
+pub fn receiveMessage(client: *Client) !InMessage.Header {
+    const Header = InMessage.Header;
+    const fifo = client.pooler.fifo(.in);
+
+    while (try client.pooler.poll()) {
+        const buf = fifo.readableSlice(0);
+        assert(fifo.readableLength() == buf.len);
+        if (buf.len >= @sizeOf(Header)) {
+            // workaround for https://github.com/ziglang/zig/issues/14904
+            const bytes_len = bswap_and_workaround_u32(buf[4..][0..4]);
+            const tag = bswap_and_workaround_tag(buf[0..][0..4]);
+
+            if (buf.len - @sizeOf(Header) >= bytes_len) {
+                fifo.discard(@sizeOf(Header));
+                return .{
+                    .tag = tag,
+                    .bytes_len = bytes_len,
+                };
+            } else {
+                const needed = bytes_len - (buf.len - @sizeOf(Header));
+                const write_buffer = try fifo.writableWithSize(needed);
+                const amt = try client.in.readAll(write_buffer);
+                fifo.update(amt);
+                continue;
+            }
+        }
+
+        const write_buffer = try fifo.writableWithSize(256);
+        const amt = try client.in.read(write_buffer);
+        fifo.update(amt);
+    }
+    return error.Timeout;
+}
+
+pub fn receiveEmitBinPath(client: *Client) !InMessage.EmitBinPath {
+    const reader = client.pooler.fifo(.in).reader();
+    return reader.readStruct(InMessage.EmitBinPath);
+}
+
+pub fn receiveErrorBundle(client: *Client) !InMessage.ErrorBundle {
+    const reader = client.pooler.fifo(.in).reader();
+    return .{
+        .extra_len = try reader.readIntLittle(u32),
+        .string_bytes_len = try reader.readIntLittle(u32),
+    };
+}
+
+pub fn receiveBytes(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u8 {
+    const reader = client.pooler.fifo(.in).reader();
+    const result = try reader.readAllAlloc(allocator, len);
+    if (result.len != len) return error.UnexpectedEOF;
+    return result;
+}
+
+pub fn receiveIntArray(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u32 {
+    const reader = client.pooler.fifo(.in).reader();
+    var array_list = std.ArrayListAligned(u8, 4).init(allocator);
+    errdefer array_list.deinit();
+    try reader.readAllArrayListAligned(4, &array_list, len);
+    const bytes = try array_list.toOwnedSlice();
+    const result = std.mem.bytesAsSlice(u32, bytes);
+    if (need_bswap) {
+        bswap_u32_array(result);
+    }
+    return result;
+}
+
+pub fn serveMessage(
+    client: *const Client,
+    header: OutMessage.Header,
+    bufs: []const []const u8,
+) !void {
+    var iovecs: [10]std.os.iovec_const = undefined;
+    const header_le = bswap(header);
+    iovecs[0] = .{
+        .iov_base = @ptrCast([*]const u8, &header_le),
+        .iov_len = @sizeOf(OutMessage.Header),
+    };
+    for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
+        iovec.* = .{
+            .iov_base = buf.ptr,
+            .iov_len = buf.len,
+        };
+    }
+    try client.out.writevAll(iovecs[0 .. bufs.len + 1]);
+}
+
+fn bswap(x: anytype) @TypeOf(x) {
+    if (!need_bswap) return x;
+
+    const T = @TypeOf(x);
+    switch (@typeInfo(T)) {
+        .Enum => return @intToEnum(T, @byteSwap(@enumToInt(x))),
+        .Int => return @byteSwap(x),
+        .Struct => |info| switch (info.layout) {
+            .Extern => {
+                var result: T = undefined;
+                inline for (info.fields) |field| {
+                    @field(result, field.name) = bswap(@field(x, field.name));
+                }
+                return result;
+            },
+            .Packed => {
+                const I = info.backing_integer.?;
+                return @bitCast(T, @byteSwap(@bitCast(I, x)));
+            },
+            .Auto => @compileError("auto layout struct"),
+        },
+        else => @compileError("bswap on type " ++ @typeName(T)),
+    }
+}
+
+fn bswap_u32_array(slice: []u32) void {
+    comptime assert(need_bswap);
+    for (slice) |*elem| elem.* = @byteSwap(elem.*);
+}
+
+/// workaround for https://github.com/ziglang/zig/issues/14904
+fn bswap_and_workaround_u32(bytes_ptr: *const [4]u8) u32 {
+    return std.mem.readIntLittle(u32, bytes_ptr);
+}
+
+/// workaround for https://github.com/ziglang/zig/issues/14904
+fn bswap_and_workaround_tag(bytes_ptr: *const [4]u8) InMessage.Tag {
+    const int = std.mem.readIntLittle(u32, bytes_ptr);
+    return @intToEnum(InMessage.Tag, int);
+}
+
+const OutMessage = std.zig.Client.Message;
+const InMessage = std.zig.Server.Message;
+
+const Client = @This();
+const builtin = @import("builtin");
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const native_endian = builtin.target.cpu.arch.endian();
+const need_bswap = native_endian != .Little;

src/translate_c.zig

@@ -6,6 +6,7 @@ const ast = @import("ast.zig");
 const tracy = @import("tracy.zig");
 const Ast = std.zig.Ast;
 const URI = @import("uri.zig");
+const ZCS = @import("ZigCompileServer.zig");
 const log = std.log.scoped(.zls_translate_c);
 
 /// converts a `@cInclude` node into an equivalent c header file
@@ -133,7 +134,12 @@ pub const Result = union(enum) {
 /// returns a URI to the generated zig file on success or the content of stderr on failure
 /// null indicates a failure which is automatically logged
 /// Caller owns returned memory.
-pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []const []const u8, source: []const u8) error{OutOfMemory}!?Result {
+pub fn translate(
+    allocator: std.mem.Allocator,
+    config: Config,
+    include_dirs: []const []const u8,
+    source: []const u8,
+) !?Result {
     const tracy_zone = tracy.trace(@src());
     defer tracy_zone.end();
 
@@ -157,12 +163,12 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c
     const base_args = &[_][]const u8{
         config.zig_exe_path orelse return null,
         "translate-c",
-        "--enable-cache",
         "--zig-lib-dir",
         config.zig_lib_path orelse return null,
         "--cache-dir",
         config.global_cache_path.?,
         "-lc",
+        "--listen=-",
     };
 
     const argc = base_args.len + 2 * include_dirs.len + 1;
@@ -178,28 +184,86 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c
 
     argv.appendAssumeCapacity(file_path);
 
-    const result = std.ChildProcess.exec(.{
+    var process = std.ChildProcess.init(argv.items, allocator);
-        .allocator = allocator,
+    process.stdin_behavior = .Pipe;
-        .argv = argv.items,
+    process.stdout_behavior = .Pipe;
-    }) catch |err| {
+    process.stderr_behavior = .Pipe;
-        log.err("Failed to execute zig translate-c process, error: {}", .{err});
+
+    errdefer |err| if (!zig_builtin.is_test) blk: {
+        const joined = std.mem.join(allocator, " ", argv.items) catch break :blk;
+        defer allocator.free(joined);
+        if (process.stderr) |stderr| {
+            const stderr_output = stderr.readToEndAlloc(allocator, std.math.maxInt(usize)) catch break :blk;
+            defer allocator.free(stderr_output);
+            log.err("failed zig translate-c command:\n{s}\nstderr:{s}\nerror:{}\n", .{ joined, stderr_output, err });
+        } else {
+            log.err("failed zig translate-c command:\n{s}\nerror:{}\n", .{ joined, err });
+        }
+    };
+
+    process.spawn() catch |err| {
+        log.err("failed to spawn zig translate-c process, error: {}", .{err});
         return null;
     };
 
-    defer allocator.free(result.stdout);
+    defer _ = process.wait() catch |wait_err| blk: {
-    defer allocator.free(result.stderr);
+        log.err("zig translate-c process did not terminate, error: {}", .{wait_err});
+        break :blk process.kill() catch |kill_err| {
+            std.debug.panic("failed to terminate zig translate-c process, error: {}", .{kill_err});
+        };
+    };
 
-    return switch (result.term) {
+    var zcs = ZCS.init(.{
-        .Exited => |code| if (code == 0) {
+        .gpa = allocator,
-            return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result.stdout, '\n')) };
+        .in = process.stdout.?,
-        } else {
+        .out = process.stdin.?,
-            return Result{ .failure = try allocator.dupe(u8, std.mem.sliceTo(result.stderr, '\n')) };
+    });
+    defer zcs.deinit();
+
+    try zcs.serveMessage(.{ .tag = .update, .bytes_len = 0 }, &.{});
+    try zcs.serveMessage(.{ .tag = .exit, .bytes_len = 0 }, &.{});
+
+    while (true) {
+        const header = try zcs.receiveMessage();
+        // log.debug("received header: {}", .{header});
+
+        switch (header.tag) {
+            .zig_version => {
+                // log.debug("zig-version: {s}", .{zcs.receive_fifo.readableSliceOfLen(header.bytes_len)});
+                zcs.pooler.fifo(.in).discard(header.bytes_len);
+            },
+            .emit_bin_path => {
+                const body_size = @sizeOf(std.zig.Server.Message.EmitBinPath);
+                if (header.bytes_len <= body_size) return error.InvalidResponse;
+
+                const trailing_size = header.bytes_len - body_size;
+
+                _ = try zcs.receiveEmitBinPath();
+
+                const result_path = try zcs.receiveBytes(allocator, trailing_size);
+                defer allocator.free(result_path);
+
+                return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result_path, '\n')) };
+            },
+            .error_bundle => {
+                const error_bundle_header = try zcs.receiveErrorBundle();
+
+                const extra = try zcs.receiveIntArray(allocator, error_bundle_header.extra_len);
+                defer allocator.free(extra);
+
+                const string_bytes = try zcs.receiveBytes(allocator, error_bundle_header.string_bytes_len);
+                defer allocator.free(string_bytes);
+
+                const error_bundle = std.zig.ErrorBundle{ .string_bytes = string_bytes, .extra = extra };
+
+                return Result{ .failure = try allocator.dupe(u8, error_bundle.getCompileLogOutput()) };
             },
             else => {
-            log.err("zig translate-c process terminated '{}'", .{result.term});
+                log.warn("received unexpected message {} from zig compile server", .{header.tag});
                 return null;
             },
-    };
+        }
+    }
 }
 
 fn extractString(str: []const u8) []const u8 {

tests/language_features/cimport.zig

@@ -8,6 +8,24 @@ const translate_c = zls.translate_c;
 
 const allocator: std.mem.Allocator = std.testing.allocator;
 
+test "zig compile server - translate c" {
+    var result1 = try testTranslate(
+        \\void foo(int);
+        \\void bar(float*);
+    );
+    defer result1.deinit(allocator);
+    try std.testing.expect(result1 == .success);
+
+    // TODO the zig compiler doesn't seem to report error bundles for translate-c
+    // Hopefully I can fix that once llvm-16 finished compiling :)
+
+    var result2 = testTranslate(
+        \\#include <this_file_doesnt_exist>
+    );
+    defer if (result2) |*r| r.deinit(allocator) else |_| {};
+    try std.testing.expectError(error.Timeout, result2);
+}
+
 test "convertCInclude - empty" {
     try testConvertCInclude("@cImport()", "");
     try testConvertCInclude("@cImport({})", "");
@@ -29,7 +47,7 @@ test "convertCInclude - cInclude" {
     );
 }
 
-test "translate_c - cDefine" {
+test "convertCInclude - cDefine" {
     try testConvertCInclude(
         \\@cImport(@cDefine("FOO", "BAR"))
     ,
@@ -42,7 +60,7 @@ test "translate_c - cDefine" {
     );
 }
 
-test "translate_c - cUndef" {
+test "convertCInclude - cUndef" {
     try testConvertCInclude(
         \\@cImport(@cUndef("FOO"))
     ,
@@ -85,3 +103,33 @@ fn testConvertCInclude(cimport_source: []const u8, expected: []const u8) !void {
 
     try std.testing.expectEqualStrings(expected, trimmed_output);
 }
+
+fn testTranslate(c_source: []const u8) !translate_c.Result {
+    if (!std.process.can_spawn) return error.SkipZigTest;
+
+    var config: zls.Config = .{};
+    defer std.json.parseFree(zls.Config, allocator, config);
+
+    var runtime_zig_version: ?zls.ZigVersionWrapper = null;
+    defer if (runtime_zig_version) |*v| v.free();
+
+    try zls.configuration.configChanged(&config, &runtime_zig_version, allocator, null);
+
+    if (config.global_cache_path == null or
+        config.zig_exe_path == null or
+        config.zig_lib_path == null) return error.SkipZigTest;
+
+    const result = (try translate_c.translate(allocator, config, &.{}, c_source)).?;
+
+    switch (result) {
+        .success => |uri| {
+            const path = try zls.URI.parse(allocator, uri);
+            defer allocator.free(path);
+            try std.testing.expect(std.fs.path.isAbsolute(path));
+        },
+        .failure => |message| {
+            try std.testing.expect(message.len != 0);
+        },
+    }
+    return result;
+}