Merge pull request #1136 from Techatrix/revive-translateC
revive C import support
This commit is contained in:
Lee Cannon
GitHub
commit
54f4768070
@ -988,7 +988,10 @@ pub fn collectIncludeDirs(
|
||||
|
||||
const native_include_dirs = try native_paths.include_dirs.toOwnedSlice();
|
||||
defer allocator.free(native_include_dirs);
|
||||
include_dirs.appendSliceAssumeCapacity(native_include_dirs);
|
||||
for (native_include_dirs) |native_include_dir| {
|
||||
var array = std.ArrayListUnmanaged(u8).fromOwnedSliceSentinel(0, native_include_dir);
|
||||
include_dirs.appendAssumeCapacity(try array.toOwnedSlice(allocator));
|
||||
}
|
||||
|
||||
for (build_file_includes_paths) |include_path| {
|
||||
const absolute_path = if (std.fs.path.isAbsolute(include_path))
|
||||
@ -1015,10 +1018,6 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index
|
||||
|
||||
if (!std.process.can_spawn) return null;
|
||||
|
||||
// FIXME: Re-enable cimport resolution once https://github.com/ziglang/zig/issues/15025 is resolved
|
||||
// Tracking issue: https://github.com/zigtools/zls/issues/1080
|
||||
if (true) return null;
|
||||
|
||||
const index = std.mem.indexOfScalar(Ast.Node.Index, handle.cimports.items(.node), node) orelse return null;
|
||||
|
||||
const hash: Hash = handle.cimports.items(.hash)[index];
|
||||
@ -1039,12 +1038,19 @@ pub fn resolveCImport(self: *DocumentStore, handle: Handle, node: Ast.Node.Index
|
||||
return null;
|
||||
};
|
||||
|
||||
var result = (try translate_c.translate(
|
||||
const maybe_result = translate_c.translate(
|
||||
self.allocator,
|
||||
self.config.*,
|
||||
include_dirs.items,
|
||||
source,
|
||||
)) orelse return null;
|
||||
) catch |err| switch (err) {
|
||||
error.OutOfMemory => |e| return e,
|
||||
else => |e| {
|
||||
log.err("failed to translate cimport: {}", .{e});
|
||||
return null;
|
||||
},
|
||||
};
|
||||
var result = maybe_result orelse return null;
|
||||
|
||||
self.cimports.putNoClobber(self.allocator, hash, result) catch result.deinit(self.allocator);
|
||||
|
||||
|
||||
168
src/ZigCompileServer.zig
Normal file
168
src/ZigCompileServer.zig
Normal file
@ -0,0 +1,168 @@
|
||||
//! modified version of https://github.com/ziglang/zig/blob/master/lib/std/zig/Server.zig
|
||||
//! I don't know why but this code seems to work
|
||||
//! zig binary serialization library in stdlib, when?
|
||||
|
||||
in: std.fs.File,
|
||||
out: std.fs.File,
|
||||
pooler: std.io.Poller(StreamEnum),
|
||||
|
||||
const StreamEnum = enum { in };
|
||||
|
||||
pub const Options = struct {
|
||||
gpa: Allocator,
|
||||
in: std.fs.File,
|
||||
out: std.fs.File,
|
||||
};
|
||||
|
||||
pub fn init(options: Options) Client {
|
||||
var s: Client = .{
|
||||
.in = options.in,
|
||||
.out = options.out,
|
||||
.pooler = std.io.poll(options.gpa, StreamEnum, .{ .in = options.in }),
|
||||
};
|
||||
return s;
|
||||
}
|
||||
|
||||
pub fn deinit(s: *Client) void {
|
||||
s.pooler.deinit();
|
||||
s.* = undefined;
|
||||
}
|
||||
|
||||
pub fn receiveMessage(client: *Client) !InMessage.Header {
|
||||
const Header = InMessage.Header;
|
||||
const fifo = client.pooler.fifo(.in);
|
||||
|
||||
while (try client.pooler.poll()) {
|
||||
const buf = fifo.readableSlice(0);
|
||||
assert(fifo.readableLength() == buf.len);
|
||||
if (buf.len >= @sizeOf(Header)) {
|
||||
// workaround for https://github.com/ziglang/zig/issues/14904
|
||||
const bytes_len = bswap_and_workaround_u32(buf[4..][0..4]);
|
||||
const tag = bswap_and_workaround_tag(buf[0..][0..4]);
|
||||
|
||||
if (buf.len - @sizeOf(Header) >= bytes_len) {
|
||||
fifo.discard(@sizeOf(Header));
|
||||
return .{
|
||||
.tag = tag,
|
||||
.bytes_len = bytes_len,
|
||||
};
|
||||
} else {
|
||||
const needed = bytes_len - (buf.len - @sizeOf(Header));
|
||||
const write_buffer = try fifo.writableWithSize(needed);
|
||||
const amt = try client.in.readAll(write_buffer);
|
||||
fifo.update(amt);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const write_buffer = try fifo.writableWithSize(256);
|
||||
const amt = try client.in.read(write_buffer);
|
||||
fifo.update(amt);
|
||||
}
|
||||
return error.Timeout;
|
||||
}
|
||||
|
||||
pub fn receiveEmitBinPath(client: *Client) !InMessage.EmitBinPath {
|
||||
const reader = client.pooler.fifo(.in).reader();
|
||||
return reader.readStruct(InMessage.EmitBinPath);
|
||||
}
|
||||
|
||||
pub fn receiveErrorBundle(client: *Client) !InMessage.ErrorBundle {
|
||||
const reader = client.pooler.fifo(.in).reader();
|
||||
return .{
|
||||
.extra_len = try reader.readIntLittle(u32),
|
||||
.string_bytes_len = try reader.readIntLittle(u32),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn receiveBytes(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u8 {
|
||||
const reader = client.pooler.fifo(.in).reader();
|
||||
const result = try reader.readAllAlloc(allocator, len);
|
||||
if (result.len != len) return error.UnexpectedEOF;
|
||||
return result;
|
||||
}
|
||||
|
||||
pub fn receiveIntArray(client: *Client, allocator: std.mem.Allocator, len: usize) ![]u32 {
|
||||
const reader = client.pooler.fifo(.in).reader();
|
||||
var array_list = std.ArrayListAligned(u8, 4).init(allocator);
|
||||
errdefer array_list.deinit();
|
||||
try reader.readAllArrayListAligned(4, &array_list, len);
|
||||
const bytes = try array_list.toOwnedSlice();
|
||||
const result = std.mem.bytesAsSlice(u32, bytes);
|
||||
if (need_bswap) {
|
||||
bswap_u32_array(result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
pub fn serveMessage(
|
||||
client: *const Client,
|
||||
header: OutMessage.Header,
|
||||
bufs: []const []const u8,
|
||||
) !void {
|
||||
var iovecs: [10]std.os.iovec_const = undefined;
|
||||
const header_le = bswap(header);
|
||||
iovecs[0] = .{
|
||||
.iov_base = @ptrCast([*]const u8, &header_le),
|
||||
.iov_len = @sizeOf(OutMessage.Header),
|
||||
};
|
||||
for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
|
||||
iovec.* = .{
|
||||
.iov_base = buf.ptr,
|
||||
.iov_len = buf.len,
|
||||
};
|
||||
}
|
||||
try client.out.writevAll(iovecs[0 .. bufs.len + 1]);
|
||||
}
|
||||
|
||||
fn bswap(x: anytype) @TypeOf(x) {
|
||||
if (!need_bswap) return x;
|
||||
|
||||
const T = @TypeOf(x);
|
||||
switch (@typeInfo(T)) {
|
||||
.Enum => return @intToEnum(T, @byteSwap(@enumToInt(x))),
|
||||
.Int => return @byteSwap(x),
|
||||
.Struct => |info| switch (info.layout) {
|
||||
.Extern => {
|
||||
var result: T = undefined;
|
||||
inline for (info.fields) |field| {
|
||||
@field(result, field.name) = bswap(@field(x, field.name));
|
||||
}
|
||||
return result;
|
||||
},
|
||||
.Packed => {
|
||||
const I = info.backing_integer.?;
|
||||
return @bitCast(T, @byteSwap(@bitCast(I, x)));
|
||||
},
|
||||
.Auto => @compileError("auto layout struct"),
|
||||
},
|
||||
else => @compileError("bswap on type " ++ @typeName(T)),
|
||||
}
|
||||
}
|
||||
|
||||
fn bswap_u32_array(slice: []u32) void {
|
||||
comptime assert(need_bswap);
|
||||
for (slice) |*elem| elem.* = @byteSwap(elem.*);
|
||||
}
|
||||
|
||||
/// workaround for https://github.com/ziglang/zig/issues/14904
|
||||
fn bswap_and_workaround_u32(bytes_ptr: *const [4]u8) u32 {
|
||||
return std.mem.readIntLittle(u32, bytes_ptr);
|
||||
}
|
||||
|
||||
/// workaround for https://github.com/ziglang/zig/issues/14904
|
||||
fn bswap_and_workaround_tag(bytes_ptr: *const [4]u8) InMessage.Tag {
|
||||
const int = std.mem.readIntLittle(u32, bytes_ptr);
|
||||
return @intToEnum(InMessage.Tag, int);
|
||||
}
|
||||
|
||||
const OutMessage = std.zig.Client.Message;
|
||||
const InMessage = std.zig.Server.Message;
|
||||
|
||||
const Client = @This();
|
||||
const builtin = @import("builtin");
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
const assert = std.debug.assert;
|
||||
const native_endian = builtin.target.cpu.arch.endian();
|
||||
const need_bswap = native_endian != .Little;
|
||||
@ -6,6 +6,7 @@ const ast = @import("ast.zig");
|
||||
const tracy = @import("tracy.zig");
|
||||
const Ast = std.zig.Ast;
|
||||
const URI = @import("uri.zig");
|
||||
const ZCS = @import("ZigCompileServer.zig");
|
||||
const log = std.log.scoped(.zls_translate_c);
|
||||
|
||||
/// converts a `@cInclude` node into an equivalent c header file
|
||||
@ -133,7 +134,12 @@ pub const Result = union(enum) {
|
||||
/// returns a URI to the generated zig file on success or the content of stderr on failure
|
||||
/// null indicates a failure which is automatically logged
|
||||
/// Caller owns returned memory.
|
||||
pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []const []const u8, source: []const u8) error{OutOfMemory}!?Result {
|
||||
pub fn translate(
|
||||
allocator: std.mem.Allocator,
|
||||
config: Config,
|
||||
include_dirs: []const []const u8,
|
||||
source: []const u8,
|
||||
) !?Result {
|
||||
const tracy_zone = tracy.trace(@src());
|
||||
defer tracy_zone.end();
|
||||
|
||||
@ -157,12 +163,12 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c
|
||||
const base_args = &[_][]const u8{
|
||||
config.zig_exe_path orelse return null,
|
||||
"translate-c",
|
||||
"--enable-cache",
|
||||
"--zig-lib-dir",
|
||||
config.zig_lib_path orelse return null,
|
||||
"--cache-dir",
|
||||
config.global_cache_path.?,
|
||||
"-lc",
|
||||
"--listen=-",
|
||||
};
|
||||
|
||||
const argc = base_args.len + 2 * include_dirs.len + 1;
|
||||
@ -178,28 +184,86 @@ pub fn translate(allocator: std.mem.Allocator, config: Config, include_dirs: []c
|
||||
|
||||
argv.appendAssumeCapacity(file_path);
|
||||
|
||||
const result = std.ChildProcess.exec(.{
|
||||
.allocator = allocator,
|
||||
.argv = argv.items,
|
||||
}) catch |err| {
|
||||
log.err("Failed to execute zig translate-c process, error: {}", .{err});
|
||||
var process = std.ChildProcess.init(argv.items, allocator);
|
||||
process.stdin_behavior = .Pipe;
|
||||
process.stdout_behavior = .Pipe;
|
||||
process.stderr_behavior = .Pipe;
|
||||
|
||||
errdefer |err| if (!zig_builtin.is_test) blk: {
|
||||
const joined = std.mem.join(allocator, " ", argv.items) catch break :blk;
|
||||
defer allocator.free(joined);
|
||||
if (process.stderr) |stderr| {
|
||||
const stderr_output = stderr.readToEndAlloc(allocator, std.math.maxInt(usize)) catch break :blk;
|
||||
defer allocator.free(stderr_output);
|
||||
log.err("failed zig translate-c command:\n{s}\nstderr:{s}\nerror:{}\n", .{ joined, stderr_output, err });
|
||||
} else {
|
||||
log.err("failed zig translate-c command:\n{s}\nerror:{}\n", .{ joined, err });
|
||||
}
|
||||
};
|
||||
|
||||
process.spawn() catch |err| {
|
||||
log.err("failed to spawn zig translate-c process, error: {}", .{err});
|
||||
return null;
|
||||
};
|
||||
|
||||
defer allocator.free(result.stdout);
|
||||
defer allocator.free(result.stderr);
|
||||
|
||||
return switch (result.term) {
|
||||
.Exited => |code| if (code == 0) {
|
||||
return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result.stdout, '\n')) };
|
||||
} else {
|
||||
return Result{ .failure = try allocator.dupe(u8, std.mem.sliceTo(result.stderr, '\n')) };
|
||||
},
|
||||
else => {
|
||||
log.err("zig translate-c process terminated '{}'", .{result.term});
|
||||
return null;
|
||||
},
|
||||
defer _ = process.wait() catch |wait_err| blk: {
|
||||
log.err("zig translate-c process did not terminate, error: {}", .{wait_err});
|
||||
break :blk process.kill() catch |kill_err| {
|
||||
std.debug.panic("failed to terminate zig translate-c process, error: {}", .{kill_err});
|
||||
};
|
||||
};
|
||||
|
||||
var zcs = ZCS.init(.{
|
||||
.gpa = allocator,
|
||||
.in = process.stdout.?,
|
||||
.out = process.stdin.?,
|
||||
});
|
||||
defer zcs.deinit();
|
||||
|
||||
try zcs.serveMessage(.{ .tag = .update, .bytes_len = 0 }, &.{});
|
||||
try zcs.serveMessage(.{ .tag = .exit, .bytes_len = 0 }, &.{});
|
||||
|
||||
while (true) {
|
||||
const header = try zcs.receiveMessage();
|
||||
// log.debug("received header: {}", .{header});
|
||||
|
||||
switch (header.tag) {
|
||||
.zig_version => {
|
||||
// log.debug("zig-version: {s}", .{zcs.receive_fifo.readableSliceOfLen(header.bytes_len)});
|
||||
zcs.pooler.fifo(.in).discard(header.bytes_len);
|
||||
},
|
||||
.emit_bin_path => {
|
||||
const body_size = @sizeOf(std.zig.Server.Message.EmitBinPath);
|
||||
if (header.bytes_len <= body_size) return error.InvalidResponse;
|
||||
|
||||
const trailing_size = header.bytes_len - body_size;
|
||||
|
||||
_ = try zcs.receiveEmitBinPath();
|
||||
|
||||
const result_path = try zcs.receiveBytes(allocator, trailing_size);
|
||||
defer allocator.free(result_path);
|
||||
|
||||
return Result{ .success = try URI.fromPath(allocator, std.mem.sliceTo(result_path, '\n')) };
|
||||
},
|
||||
.error_bundle => {
|
||||
const error_bundle_header = try zcs.receiveErrorBundle();
|
||||
|
||||
const extra = try zcs.receiveIntArray(allocator, error_bundle_header.extra_len);
|
||||
defer allocator.free(extra);
|
||||
|
||||
const string_bytes = try zcs.receiveBytes(allocator, error_bundle_header.string_bytes_len);
|
||||
defer allocator.free(string_bytes);
|
||||
|
||||
const error_bundle = std.zig.ErrorBundle{ .string_bytes = string_bytes, .extra = extra };
|
||||
|
||||
return Result{ .failure = try allocator.dupe(u8, error_bundle.getCompileLogOutput()) };
|
||||
},
|
||||
else => {
|
||||
log.warn("received unexpected message {} from zig compile server", .{header.tag});
|
||||
return null;
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extractString(str: []const u8) []const u8 {
|
||||
|
||||
@ -8,6 +8,24 @@ const translate_c = zls.translate_c;
|
||||
|
||||
const allocator: std.mem.Allocator = std.testing.allocator;
|
||||
|
||||
test "zig compile server - translate c" {
|
||||
var result1 = try testTranslate(
|
||||
\\void foo(int);
|
||||
\\void bar(float*);
|
||||
);
|
||||
defer result1.deinit(allocator);
|
||||
try std.testing.expect(result1 == .success);
|
||||
|
||||
// TODO the zig compiler doesn't seem to report error bundles for translate-c
|
||||
// Hopefully I can fix that once llvm-16 finished compiling :)
|
||||
|
||||
var result2 = testTranslate(
|
||||
\\#include <this_file_doesnt_exist>
|
||||
);
|
||||
defer if (result2) |*r| r.deinit(allocator) else |_| {};
|
||||
try std.testing.expectError(error.Timeout, result2);
|
||||
}
|
||||
|
||||
test "convertCInclude - empty" {
|
||||
try testConvertCInclude("@cImport()", "");
|
||||
try testConvertCInclude("@cImport({})", "");
|
||||
@ -29,7 +47,7 @@ test "convertCInclude - cInclude" {
|
||||
);
|
||||
}
|
||||
|
||||
test "translate_c - cDefine" {
|
||||
test "convertCInclude - cDefine" {
|
||||
try testConvertCInclude(
|
||||
\\@cImport(@cDefine("FOO", "BAR"))
|
||||
,
|
||||
@ -42,7 +60,7 @@ test "translate_c - cDefine" {
|
||||
);
|
||||
}
|
||||
|
||||
test "translate_c - cUndef" {
|
||||
test "convertCInclude - cUndef" {
|
||||
try testConvertCInclude(
|
||||
\\@cImport(@cUndef("FOO"))
|
||||
,
|
||||
@ -85,3 +103,33 @@ fn testConvertCInclude(cimport_source: []const u8, expected: []const u8) !void {
|
||||
|
||||
try std.testing.expectEqualStrings(expected, trimmed_output);
|
||||
}
|
||||
|
||||
fn testTranslate(c_source: []const u8) !translate_c.Result {
|
||||
if (!std.process.can_spawn) return error.SkipZigTest;
|
||||
|
||||
var config: zls.Config = .{};
|
||||
defer std.json.parseFree(zls.Config, allocator, config);
|
||||
|
||||
var runtime_zig_version: ?zls.ZigVersionWrapper = null;
|
||||
defer if (runtime_zig_version) |*v| v.free();
|
||||
|
||||
try zls.configuration.configChanged(&config, &runtime_zig_version, allocator, null);
|
||||
|
||||
if (config.global_cache_path == null or
|
||||
config.zig_exe_path == null or
|
||||
config.zig_lib_path == null) return error.SkipZigTest;
|
||||
|
||||
const result = (try translate_c.translate(allocator, config, &.{}, c_source)).?;
|
||||
|
||||
switch (result) {
|
||||
.success => |uri| {
|
||||
const path = try zls.URI.parse(allocator, uri);
|
||||
defer allocator.free(path);
|
||||
try std.testing.expect(std.fs.path.isAbsolute(path));
|
||||
},
|
||||
.failure => |message| {
|
||||
try std.testing.expect(message.len != 0);
|
||||
},
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user