Merge pull request #1136 from Techatrix/revive-translateC

revive C import support
This commit is contained in:
Lee Cannon 2023-05-30 17:32:14 +01:00 committed by GitHub
commit 54f4768070
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 315 additions and 29 deletions

View File

@ -988,7 +988,10 @@ pub fn collectIncludeDirs(
const native_include_dirs = try native_paths.include_dirs.toOwnedSlice(); const native_include_dirs = try native_paths.include_dirs.toOwnedSlice();
defer allocator.free(native_include_dirs); defer allocator.free(native_include_dirs);
include_dirs.appendSliceAssumeCapacity(native_include_dirs); for (native_include_dirs) |native_include_dir| {
var array = std.ArrayListUnmanaged(u8).fromOwnedSliceSentinel(0, native_include_dir);
include_dirs.appendAssumeCapacity(try array.toOwnedSlice(allocator));
}
for (build_file_includes_paths) |include_path| { for (build_file_includes_paths) |include_path| {
const absolute_path = if (std.fs.path.isAbsolute(include_path)) const absolute_path = if (std.fs.path.isAbsolute(include_path))
@ -1015,10 +1018,6 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index
if (!std.process.can_spawn) return null; if (!std.process.can_spawn) return null;
// FIXME: Re-enable cimport resolution once https://github.com/ziglang/zig/issues/15025 is resolved
// Tracking issue: https://github.com/zigtools/zls/issues/1080
if (true) return null;
const index = std.mem.indexOfScalar(Ast.Node.Index, handle.cimports.items(.node), node) orelse return null; const index = std.mem.indexOfScalar(Ast.Node.Index, handle.cimports.items(.node), node) orelse return null;
const hash: Hash = handle.cimports.items(.hash)[index]; const hash: Hash = handle.cimports.items(.hash)[index];
@ -1039,12 +1038,19 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index
return null; return null;
}; };
var result = (try translate_c.translate( const maybe_result = translate_c.translate(
self.allocator, self.allocator,
self.config.*, self.config.*,
include_dirs.items, include_dirs.items,
source, source,
)) orelse return null; ) catch |err| switch (err) {
error.OutOfMemory => |e| return e,
else => |e| {
log.err("failed to translate cimport: {}", .{e});
return null;
},
};
var result = maybe_result orelse return null;
self.cimports.putNoClobber(self.allocator, hash, result) catch result.deinit(self.allocator); self.cimports.putNoClobber(self.allocator, hash, result) catch result.deinit(self.allocator);

168
src/ZigCompileServer.zig Normal file
View File

@ -0,0 +1,168 @@
//! modified version of https://github.com/ziglang/zig/blob/master/lib/std/zig/Server.zig
//! I don't know why but this code seems to work
//! zig binary serialization library in stdlib, when?
in: std.fs.File,
out: std.fs.File,
pooler: std.io.Poller(StreamEnum),
const StreamEnum = enum { in };
pub const Options = struct {
gpa: Allocator,
in: std.fs.File,
out: std.fs.File,
};
pub fn init(options: Options) Client {
var s: Client = .{
.in = options.in,
.out = options.out,
.pooler = std.io.poll(options.gpa, StreamEnum, .{ .in = options.in }),
};
return s;
}
pub fn deinit(s: *Client) void {
s.pooler.deinit();
s.* = undefined;
}
pub fn receiveMessage(client: *Client) !InMessage.Header {
const Header = InMessage.Header;
const fifo = client.pooler.fifo(.in);
while (try client.pooler.poll()) {
const buf = fifo.readableSlice(0);
assert(fifo.readableLength() == buf.len);
if (buf.len >= @sizeOf(Header)) {
// workaround for https://github.com/ziglang/zig/issues/14904
const bytes_len = bswap_and_workaround_u32(buf[4..][0..4]);
const tag = bswap_and_workaround_tag(buf[0..][0..4]);
if (buf.len - @sizeOf(Header) >= bytes_len) {
fifo.discard(@sizeOf(Header));
return .{
.tag = tag,
.bytes_len = bytes_len,
};
} else {
const needed = bytes_len - (buf.len - @sizeOf(Header));
const write_buffer = try fifo.writableWithSize(needed);
const amt = try client.in.readAll(write_buffer);
fifo.update(amt);
continue;
}
}
const write_buffer = try fifo.writableWithSize(256);
const amt = try client.in.read(write_buffer);
fifo.update(amt);
}
return error.Timeout;
}
pub fn receiveEmitBinPath(client: *Client) !InMessage.EmitBinPath {
const reader = client.pooler.fifo(.in).reader();
return reader.readStruct(InMessage.EmitBinPath);
}
pub fn receiveErrorBundle(client: *Client) !InMessage.ErrorBundle {
const reader = client.pooler.fifo(.in).reader();
return .{
.extra_len = try reader.readIntLittle(u32),
.string_bytes_len = try reader.readIntLittle(u32),
};
}
pub fn receiveBytes(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u8 {
const reader = client.pooler.fifo(.in).reader();
const result = try reader.readAllAlloc(allocator, len);
if (result.len != len) return error.UnexpectedEOF;
return result;
}
pub fn receiveIntArray(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u32 {
const reader = client.pooler.fifo(.in).reader();
var array_list = std.ArrayListAligned(u8, 4).init(allocator);
errdefer array_list.deinit();
try reader.readAllArrayListAligned(4, &array_list, len);
const bytes = try array_list.toOwnedSlice();
const result = std.mem.bytesAsSlice(u32, bytes);
if (need_bswap) {
bswap_u32_array(result);
}
return result;
}
pub fn serveMessage(
client: *const Client,
header: OutMessage.Header,
bufs: []const []const u8,
) !void {
var iovecs: [10]std.os.iovec_const = undefined;
const header_le = bswap(header);
iovecs[0] = .{
.iov_base = @ptrCast([*]const u8, &header_le),
.iov_len = @sizeOf(OutMessage.Header),
};
for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
iovec.* = .{
.iov_base = buf.ptr,
.iov_len = buf.len,
};
}
try client.out.writevAll(iovecs[0 .. bufs.len + 1]);
}
fn bswap(x: anytype) @TypeOf(x) {
if (!need_bswap) return x;
const T = @TypeOf(x);
switch (@typeInfo(T)) {
.Enum => return @intToEnum(T, @byteSwap(@enumToInt(x))),
.Int => return @byteSwap(x),
.Struct => |info| switch (info.layout) {
.Extern => {
var result: T = undefined;
inline for (info.fields) |field| {
@field(result, field.name) = bswap(@field(x, field.name));
}
return result;
},
.Packed => {
const I = info.backing_integer.?;
return @bitCast(T, @byteSwap(@bitCast(I, x)));
},
.Auto => @compileError("auto layout struct"),
},
else => @compileError("bswap on type " ++ @typeName(T)),
}
}
fn bswap_u32_array(slice: []u32) void {
comptime assert(need_bswap);
for (slice) |*elem| elem.* = @byteSwap(elem.*);
}
/// workaround for https://github.com/ziglang/zig/issues/14904
fn bswap_and_workaround_u32(bytes_ptr: *const [4]u8) u32 {
return std.mem.readIntLittle(u32, bytes_ptr);
}
/// workaround for https://github.com/ziglang/zig/issues/14904
fn bswap_and_workaround_tag(bytes_ptr: *const [4]u8) InMessage.Tag {
const int = std.mem.readIntLittle(u32, bytes_ptr);
return @intToEnum(InMessage.Tag, int);
}
const OutMessage = std.zig.Client.Message;
const InMessage = std.zig.Server.Message;
const Client = @This();
const builtin = @import("builtin");
const std = @import("std");
const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const native_endian = builtin.target.cpu.arch.endian();
const need_bswap = native_endian != .Little;

View File

@ -6,6 +6,7 @@ const ast = @import("ast.zig");
const tracy = @import("tracy.zig"); const tracy = @import("tracy.zig");
const Ast = std.zig.Ast; const Ast = std.zig.Ast;
const URI = @import("uri.zig"); const URI = @import("uri.zig");
const ZCS = @import("ZigCompileServer.zig");
const log = std.log.scoped(.zls_translate_c); const log = std.log.scoped(.zls_translate_c);
/// converts a `@cInclude` node into an equivalent c header file /// converts a `@cInclude` node into an equivalent c header file
@ -133,7 +134,12 @@ pub const Result = union(enum) {
/// returns a URI to the generated zig file on success or the content of stderr on failure /// returns a URI to the generated zig file on success or the content of stderr on failure
/// null indicates a failure which is automatically logged /// null indicates a failure which is automatically logged
/// Caller owns returned memory. /// Caller owns returned memory.
pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []const []const u8, source: []const u8) error{OutOfMemory}!?Result { pub fn translate(
allocator: std.mem.Allocator,
config: Config,
include_dirs: []const []const u8,
source: []const u8,
) !?Result {
const tracy_zone = tracy.trace(@src()); const tracy_zone = tracy.trace(@src());
defer tracy_zone.end(); defer tracy_zone.end();
@ -157,12 +163,12 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c
const base_args = &[_][]const u8{ const base_args = &[_][]const u8{
config.zig_exe_path orelse return null, config.zig_exe_path orelse return null,
"translate-c", "translate-c",
"--enable-cache",
"--zig-lib-dir", "--zig-lib-dir",
config.zig_lib_path orelse return null, config.zig_lib_path orelse return null,
"--cache-dir", "--cache-dir",
config.global_cache_path.?, config.global_cache_path.?,
"-lc", "-lc",
"--listen=-",
}; };
const argc = base_args.len + 2 * include_dirs.len + 1; const argc = base_args.len + 2 * include_dirs.len + 1;
@ -178,28 +184,86 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c
argv.appendAssumeCapacity(file_path); argv.appendAssumeCapacity(file_path);
const result = std.ChildProcess.exec(.{ var process = std.ChildProcess.init(argv.items, allocator);
.allocator = allocator, process.stdin_behavior = .Pipe;
.argv = argv.items, process.stdout_behavior = .Pipe;
}) catch |err| { process.stderr_behavior = .Pipe;
log.err("Failed to execute zig translate-c process, error: {}", .{err});
errdefer |err| if (!zig_builtin.is_test) blk: {
const joined = std.mem.join(allocator, " ", argv.items) catch break :blk;
defer allocator.free(joined);
if (process.stderr) |stderr| {
const stderr_output = stderr.readToEndAlloc(allocator, std.math.maxInt(usize)) catch break :blk;
defer allocator.free(stderr_output);
log.err("failed zig translate-c command:\n{s}\nstderr:{s}\nerror:{}\n", .{ joined, stderr_output, err });
} else {
log.err("failed zig translate-c command:\n{s}\nerror:{}\n", .{ joined, err });
}
};
process.spawn() catch |err| {
log.err("failed to spawn zig translate-c process, error: {}", .{err});
return null; return null;
}; };
defer allocator.free(result.stdout); defer _ = process.wait() catch |wait_err| blk: {
defer allocator.free(result.stderr); log.err("zig translate-c process did not terminate, error: {}", .{wait_err});
break :blk process.kill() catch |kill_err| {
std.debug.panic("failed to terminate zig translate-c process, error: {}", .{kill_err});
};
};
return switch (result.term) { var zcs = ZCS.init(.{
.Exited => |code| if (code == 0) { .gpa = allocator,
return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result.stdout, '\n')) }; .in = process.stdout.?,
} else { .out = process.stdin.?,
return Result{ .failure = try allocator.dupe(u8, std.mem.sliceTo(result.stderr, '\n')) }; });
defer zcs.deinit();
try zcs.serveMessage(.{ .tag = .update, .bytes_len = 0 }, &.{});
try zcs.serveMessage(.{ .tag = .exit, .bytes_len = 0 }, &.{});
while (true) {
const header = try zcs.receiveMessage();
// log.debug("received header: {}", .{header});
switch (header.tag) {
.zig_version => {
// log.debug("zig-version: {s}", .{zcs.receive_fifo.readableSliceOfLen(header.bytes_len)});
zcs.pooler.fifo(.in).discard(header.bytes_len);
},
.emit_bin_path => {
const body_size = @sizeOf(std.zig.Server.Message.EmitBinPath);
if (header.bytes_len <= body_size) return error.InvalidResponse;
const trailing_size = header.bytes_len - body_size;
_ = try zcs.receiveEmitBinPath();
const result_path = try zcs.receiveBytes(allocator, trailing_size);
defer allocator.free(result_path);
return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result_path, '\n')) };
},
.error_bundle => {
const error_bundle_header = try zcs.receiveErrorBundle();
const extra = try zcs.receiveIntArray(allocator, error_bundle_header.extra_len);
defer allocator.free(extra);
const string_bytes = try zcs.receiveBytes(allocator, error_bundle_header.string_bytes_len);
defer allocator.free(string_bytes);
const error_bundle = std.zig.ErrorBundle{ .string_bytes = string_bytes, .extra = extra };
return Result{ .failure = try allocator.dupe(u8, error_bundle.getCompileLogOutput()) };
}, },
else => { else => {
log.err("zig translate-c process terminated '{}'", .{result.term}); log.warn("received unexpected message {} from zig compile server", .{header.tag});
return null; return null;
}, },
}; }
}
} }
fn extractString(str: []const u8) []const u8 { fn extractString(str: []const u8) []const u8 {

View File

@ -8,6 +8,24 @@ const translate_c = zls.translate_c;
const allocator: std.mem.Allocator = std.testing.allocator; const allocator: std.mem.Allocator = std.testing.allocator;
test "zig compile server - translate c" {
var result1 = try testTranslate(
\\void foo(int);
\\void bar(float*);
);
defer result1.deinit(allocator);
try std.testing.expect(result1 == .success);
// TODO the zig compiler doesn't seem to report error bundles for translate-c
// Hopefully I can fix that once llvm-16 finished compiling :)
var result2 = testTranslate(
\\#include <this_file_doesnt_exist>
);
defer if (result2) |*r| r.deinit(allocator) else |_| {};
try std.testing.expectError(error.Timeout, result2);
}
test "convertCInclude - empty" { test "convertCInclude - empty" {
try testConvertCInclude("@cImport()", ""); try testConvertCInclude("@cImport()", "");
try testConvertCInclude("@cImport({})", ""); try testConvertCInclude("@cImport({})", "");
@ -29,7 +47,7 @@ test "convertCInclude - cInclude" {
); );
} }
test "translate_c - cDefine" { test "convertCInclude - cDefine" {
try testConvertCInclude( try testConvertCInclude(
\\@cImport(@cDefine("FOO", "BAR")) \\@cImport(@cDefine("FOO", "BAR"))
, ,
@ -42,7 +60,7 @@ test "translate_c - cDefine" {
); );
} }
test "translate_c - cUndef" { test "convertCInclude - cUndef" {
try testConvertCInclude( try testConvertCInclude(
\\@cImport(@cUndef("FOO")) \\@cImport(@cUndef("FOO"))
, ,
@ -85,3 +103,33 @@ fn testConvertCInclude(cimport_source: []const u8, expected: []const u8) !void {
try std.testing.expectEqualStrings(expected, trimmed_output); try std.testing.expectEqualStrings(expected, trimmed_output);
} }
fn testTranslate(c_source: []const u8) !translate_c.Result {
if (!std.process.can_spawn) return error.SkipZigTest;
var config: zls.Config = .{};
defer std.json.parseFree(zls.Config, allocator, config);
var runtime_zig_version: ?zls.ZigVersionWrapper = null;
defer if (runtime_zig_version) |*v| v.free();
try zls.configuration.configChanged(&config, &runtime_zig_version, allocator, null);
if (config.global_cache_path == null or
config.zig_exe_path == null or
config.zig_lib_path == null) return error.SkipZigTest;
const result = (try translate_c.translate(allocator, config, &.{}, c_source)).?;
switch (result) {
.success => |uri| {
const path = try zls.URI.parse(allocator, uri);
defer allocator.free(path);
try std.testing.expect(std.fs.path.isAbsolute(path));
},
.failure => |message| {
try std.testing.expect(message.len != 0);
},
}
return result;
}