Better builtin data format and script

README.md

@@ -37,7 +37,7 @@ The `zls` executable will be saved to `zls\zig-cache\bin`.
 
 | Option | Type | Default Value | What it Does |
 | --- | --- | --- | --- |
-| `-Ddata_version` | `string` (master or 0.6.0) | 0.6.0 | The data file version. This selects the files in the `src/data` folder that correspond to the Zig version being served.|
+| `-Ddata_version` | `string` (master or 0.6.0) | master | The data file version. This selects the files in the `src/data` folder that correspond to the Zig version being served.|
 
 Then, you can use the `zls` executable in an editor of your choice that has a Zig language server client!
 

build.zig

@@ -139,7 +139,7 @@ pub fn build(b: *std.build.Builder) !void {
     exe.addBuildOption(
         []const u8,
         "data_version",
-        b.option([]const u8, "data_version", "The data version - either 0.6.0 or master.") orelse "0.6.0",
+        b.option([]const u8, "data_version", "The data version - either 0.6.0 or master.") orelse "master",
     );
 
     exe.addPackage(.{ .name = "known-folders", .path = "src/known-folders/known-folders.zig" });

src/data/SCRIPTS.md

@@ -1,71 +0,0 @@
-# Data Scripts
-
-## Snippet Generation
-```js
-[...document.querySelector("#toc-Builtin-Functions").parentElement.lastElementChild.children].map(_ => {
-
-const code = document.querySelector("#" + _.innerText.slice(1)).nextElementSibling.children[0].innerText;
-var l = (code.lastIndexOf(") ") == -1 ? code.length : code.lastIndexOf(") ")) + 1
-var p = code.slice(0, l);
-
-var name = p.slice(0, p.indexOf("("));
-var body = p.slice(p.indexOf("(") + 1, -1);
-if (body.trim().length === 0) return `${name}()`;
-var nb = "";
-let depth = 0;
-let vi = 2;
-let i = 0;
-let skip = false;
-for (const c of body) {
-if (skip) {
-skip = false;
-if (c === " ") {i++; continue;}
-}
-if (c === "(") depth++;
-else if (c === ")") depth--;
-
-if (c === "," && depth == 0) {
-nb += `}, \${${vi}:`;
-vi++;
-skip = true;
-} else if (i === body.length - 1) {
-nb += c;
-nb += "}";
-} else nb += c;
-i++;
-}
-return `${name}(\${1:${nb})`;
-
-}).map(_ => JSON.stringify(_)).join(",\n");
-```
-
-## Function Signature / Details
-```js
-[...document.querySelector("#toc-Builtin-Functions").parentElement.lastElementChild.children].map(_ => {
-return document.querySelector("#" + _.innerText.slice(1)).nextElementSibling.innerText;
-}).map(_ => JSON.stringify(_)).join(",\n");
-```
-
-## Docs
-```js
-[...document.querySelector("#toc-Builtin-Functions").parentElement.lastElementChild.children].map(_ => {
-return document.querySelector("#" + _.innerText.slice(1)).nextElementSibling.nextElementSibling.innerText;
-}).map(_ => JSON.stringify(_)).join(",\n");
-```
-
-## All together now
-```js
-`/// Builtin functions
-pub const builtins = [_][]const u8{` + [...document.querySelector("#toc-Builtin-Functions").parentElement.lastElementChild.children].map(_ => {const code = document.querySelector("#" + _.innerText.slice(1)).nextElementSibling.children[0].innerText; var l = (code.lastIndexOf(") ") == -1 ? code.length : code.lastIndexOf(") ")) + 1; var p = code.slice(0, l); var name = p.slice(0, p.indexOf("("));var body = p.slice(p.indexOf("(") + 1, -1);if (body.trim().length === 0) return `${name}()`; var nb = ""; let depth = 0; let vi = 2; let i = 0; let skip = false; for (const c of body) {if (skip) {skip = false;if (c === " ") {i++; continue;};};if (c === "(") depth++;else if (c === ")") depth--;if (c === "," && depth == 0) {nb += `}, \${${vi}:`;vi++;skip = true;} else if (i === body.length - 1) {nb += c;nb += "}";} else nb += c;i++;};return `${name}(\${1:${nb})`;}).map(_ => JSON.stringify(_)).join(",\n") + `};
-
-/// Builtin function details
-pub const builtin_details = [_][]const u8{` + [...document.querySelector("#toc-Builtin-Functions").parentElement.lastElementChild.children].map(_ => {
-return document.querySelector("#" + _.innerText.slice(1)).nextElementSibling.innerText;
-}).map(_ => JSON.stringify(_)).join(",\n") + `};
-
-/// Builtin function docs
-pub const builtin_docs = [_][]const u8{` + [...document.querySelector("#toc-Builtin-Functions").parentElement.lastElementChild.children].map(_ => {
-return document.querySelector("#" + _.innerText.slice(1)).nextElementSibling.nextElementSibling.innerText;
-}).map(_ => JSON.stringify(_)).join(",\n") + `};
-`
-```

src/data/generate-data.js

@@ -0,0 +1,94 @@
+// Run this in a Chrome developer console.
+const builtins = $$("a#toc-Builtin-Functions+ul > li").map(element => {
+    const anchor = element.querySelector("a").getAttribute("href");
+    const code = $(`${anchor}+pre > code`).textContent.replace(/(\r\n|\n|\r)/gm, "");
+
+    var curr_paragraph = $(`${anchor}+pre+p`);
+    var doc = "";
+    var first = true;
+
+    while (curr_paragraph.nodeName == "P" || curr_paragraph.nodeName == "PRE") {
+        if (curr_paragraph.innerHTML == "See also:")
+            break;
+
+        if (!first) {
+            doc += "\n";
+        } else {
+            first = false;
+        }
+
+        if (curr_paragraph.nodeName == "PRE") {
+            doc += "```zig\n";
+            curr_paragraph.childNodes[0].childNodes.forEach(elem => {
+                doc += elem.textContent;
+            });
+            doc += "\n```";
+        } else {
+            curr_paragraph.childNodes.forEach(elem => {
+                doc += elem.textContent.replace(/(\s\s+)/gm, " ");
+            });
+        }
+
+        curr_paragraph = curr_paragraph.nextElementSibling;
+    }
+    return { "name": anchor.substring(1), "code": code, "documentation": doc };
+});
+
+// Take output and paste into a .zig file
+console.log(
+    `const Builtin = struct {
+    name: []const u8,
+    signature: []const u8,
+    snippet: []const u8,
+    documentation: []const u8,
+};
+
+pub const builtins = [_]Builtin{` +
+    '\n' + builtins.map(builtin => {
+        // Make a snippet
+        const first_paren_idx = builtin.code.indexOf('(');
+        var snippet = builtin.code.substr(0, first_paren_idx + 1);
+        var rest = builtin.code.substr(first_paren_idx + 1);
+
+        if (rest[0] == ')') {
+            snippet += ')';
+        } else {
+            snippet += "${1:"
+
+            var arg_idx = 2;
+            var paren_depth = 1;
+            var skip_space = false;
+            for (const char of rest) {
+                if (char == '(') {
+                    paren_depth += 1;
+                } else if (char == ')') {
+                    paren_depth -= 1;
+                    if (paren_depth == 0) {
+                        snippet += "})";
+                        break;
+                    }
+                } else if (char == '"') {
+                    snippet += "\\\"";
+                    continue;
+                } else if (char == ',' && paren_depth == 1) {
+                    snippet += "}, ${" + arg_idx + ':';
+                    arg_idx += 1;
+                    skip_space = true;
+                    continue;
+                } else if (char == ' ' && skip_space) {
+                    continue;
+                }
+
+                snippet += char;
+                skip_space = false;
+            }
+        }
+
+        return `    .{
+        .name = "@${builtin.name}",
+        .signature = "${builtin.code.replaceAll('"', "\\\"")}",
+        .snippet = "${snippet}",
+        .documentation =
+        \\\\${builtin.documentation.split('\n').join("\n        \\\\") + '\n'}    },`;
+    }).join('\n') + "\n};\n"
+);

src/main.zig

@@ -913,6 +913,42 @@ fn completeLabel(arena: *std.heap.ArenaAllocator, id: types.RequestId, pos_index
     });
 }
 
+var builtin_completions: ?[]types.CompletionItem = null;
+fn completeBuiltin(arena: *std.heap.ArenaAllocator, id: types.RequestId, config: Config) !void {
+    if (builtin_completions == null) {
+        builtin_completions = try allocator.alloc(types.CompletionItem, data.builtins.len);
+        for (data.builtins) |builtin, idx| {
+            builtin_completions.?[idx] = types.CompletionItem{
+                .label = builtin.name,
+                .kind = .Function,
+                .filterText = builtin.name[1..],
+                .detail = builtin.signature,
+                .documentation = .{
+                    .kind = .Markdown,
+                    .value = builtin.documentation,
+                },
+            };
+
+            if (config.enable_snippets) {
+                builtin_completions.?[idx].insertText = builtin.snippet[1..];
+                builtin_completions.?[idx].insertTextFormat = .Snippet;
+            } else {
+                builtin_completions.?[idx].insertText = builtin.name[1..];
+            }
+        }
+    }
+
+    try send(arena, types.Response{
+        .id = id,
+        .result = .{
+            .CompletionList = .{
+                .isIncomplete = false,
+                .items = builtin_completions.?,
+            },
+        },
+    });
+}
+
 fn completeGlobal(arena: *std.heap.ArenaAllocator, id: types.RequestId, pos_index: usize, handle: *DocumentStore.Handle, config: Config) !void {
     var completions = std.ArrayList(types.CompletionItem).init(&arena.allocator);
 
@@ -967,41 +1003,6 @@ fn documentSymbol(arena: *std.heap.ArenaAllocator, id: types.RequestId, handle:
     });
 }
 
-// Compute builtin completions at comptime.
-const builtin_completions = block: {
-    @setEvalBranchQuota(10_000);
-    const CompletionList = [data.builtins.len]types.CompletionItem;
-    var with_snippets: CompletionList = undefined;
-    var without_snippets: CompletionList = undefined;
-
-    for (data.builtins) |builtin, i| {
-        const cutoff = std.mem.indexOf(u8, builtin, "(") orelse builtin.len;
-
-        const base_completion = types.CompletionItem{
-            .label = builtin[0..cutoff],
-            .kind = .Function,
-
-            .filterText = builtin[1..cutoff],
-            .detail = data.builtin_details[i],
-            .documentation = .{
-                .kind = .Markdown,
-                .value = data.builtin_docs[i],
-            },
-        };
-
-        with_snippets[i] = base_completion;
-        with_snippets[i].insertText = builtin[1..];
-        with_snippets[i].insertTextFormat = .Snippet;
-
-        without_snippets[i] = base_completion;
-        without_snippets[i].insertText = builtin[1..cutoff];
-    }
-
-    break :block [2]CompletionList{
-        without_snippets, with_snippets,
-    };
-};
-
 fn loadConfig(folder_path: []const u8) ?Config {
     var folder = std.fs.cwd().openDir(folder_path, .{}) catch return null;
     defer folder.close();
@@ -1206,15 +1207,7 @@ fn completionHandler(arena: *std.heap.ArenaAllocator, id: types.RequestId, req:
         const this_config = configFromUriOr(req.params.textDocument.uri, config);
         const use_snippets = this_config.enable_snippets and client_capabilities.supports_snippets;
         switch (pos_context) {
-            .builtin => try send(arena, types.Response{
-                .id = id,
-                .result = .{
-                    .CompletionList = .{
-                        .isIncomplete = false,
-                        .items = builtin_completions[@boolToInt(use_snippets)][0..],
-                    },
-                },
-            }),
+            .builtin => try completeBuiltin(arena, id, this_config),
             .var_access, .empty => try completeGlobal(arena, id, doc_position.absolute_index, handle, this_config),
             .field_access => |range| try completeFieldAccess(arena, id, handle, doc_position, range, this_config),
             .global_error_set => try send(arena, types.Response{
@@ -1644,4 +1637,8 @@ pub fn main() anyerror!void {
         arena.deinit();
         arena.state = .{};
     }
+
+    if (builtin_completions) |compls| {
+        allocator.free(compls);
+    }
 }

src/types.zig

@@ -193,7 +193,8 @@ pub const MarkupKind = enum(u1) {
 };
 
 pub const MarkupContent = struct {
-    kind: MarkupKind = MarkupKind.Markdown, value: String
+    kind: MarkupKind = MarkupKind.Markdown,
+    value: String,
 };
 
 // pub const TextDocumentIdentifier = struct {