authorgravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2021-06-07 05:34:17 -07:00
committergravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2021-06-07 05:34:17 -07:00
logeaf70895de1efa8d01d58268919c4f7737554824
tree2471983bb767f3af77f7143e32b69537f99eb075
parent2076b1bd3dfc651cbe4bad90aac9e8a59454ad0a

lib/yaml- rework to be O(n) and much more memory efficient

1 files changed, 110 insertions(+), 152 deletions(-)

src/util/yaml.zig+110-152
...@@ -1,4 +1,5 @@...@@ -1,4 +1,5 @@
1const std = @import("std");1const std = @import("std");
2const yaml = @import("yaml");
23
3const c = @cImport({4const c = @cImport({
4 @cInclude("yaml.h");5 @cInclude("yaml.h");
...@@ -10,6 +11,10 @@ const u = @import("./index.zig");...@@ -10,6 +11,10 @@ const u = @import("./index.zig");
1011
11const Array = []const []const u8;12const Array = []const []const u8;
1213
14pub const Stream = struct {
15 docs: []const Document,
16};
17
13pub const Document = struct {18pub const Document = struct {
14 mapping: Mapping,19 mapping: Mapping,
15};20};
...@@ -21,11 +26,15 @@ pub const Item = union(enum) {...@@ -21,11 +26,15 @@ pub const Item = union(enum) {
21 sequence: Sequence,26 sequence: Sequence,
22 document: Document,27 document: Document,
23 string: []const u8,28 string: []const u8,
29 stream: Stream,
2430
25 pub fn format(self: Item, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) @TypeOf(writer).Error!void {31 pub fn format(self: Item, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) @TypeOf(writer).Error!void {
26 try writer.writeAll("Item{");32 try writer.writeAll("Item{");
27 switch (self) {33 switch (self) {
28 .event, .kv, .document => {34 .event => {
35 try std.fmt.format(writer, "{s}", .{@tagName(self.event.type)});
36 },
37 .kv, .document => {
29 unreachable;38 unreachable;
30 },39 },
31 .mapping => {40 .mapping => {
...@@ -121,6 +130,8 @@ pub const Mapping = struct {...@@ -121,6 +130,8 @@ pub const Mapping = struct {
121 }130 }
122};131};
123132
133pub const TokenList = []const c.yaml_event_t;
134
124//135//
125//136//
126137
...@@ -132,7 +143,7 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {...@@ -132,7 +143,7 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {
132143
133 _ = c.yaml_parser_set_input_string(&parser, input.ptr, input.len);144 _ = c.yaml_parser_set_input_string(&parser, input.ptr, input.len);
134145
135 const all_events = &std.ArrayList(Item).init(alloc);146 const all_events = &std.ArrayList(c.yaml_event_t).init(alloc);
136 var event: c.yaml_event_t = undefined;147 var event: c.yaml_event_t = undefined;
137 while (true) {148 while (true) {
138 const p = c.yaml_parser_parse(&parser, &event);149 const p = c.yaml_parser_parse(&parser, &event);
...@@ -141,7 +152,7 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {...@@ -141,7 +152,7 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {
141 }152 }
142153
143 const et = @enumToInt(event.type);154 const et = @enumToInt(event.type);
144 try all_events.append(.{ .event = event });155 try all_events.append(event);
145 c.yaml_event_delete(&event);156 c.yaml_event_delete(&event);
146157
147 if (et == c.YAML_STREAM_END_EVENT) {158 if (et == c.YAML_STREAM_END_EVENT) {
...@@ -151,179 +162,126 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {...@@ -151,179 +162,126 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {
151162
152 c.yaml_parser_delete(&parser);163 c.yaml_parser_delete(&parser);
153164
154 var l: usize = all_events.items.len;165 const p = &Parser{
155 while (true) {166 .alloc = alloc,
156 try condense_event_list(all_events, lines);167 .tokens = all_events.items,
157 if (l == all_events.items.len) {168 .lines = lines,
158 break;169 .index = 0,
159 }170 };
160 l = all_events.items.len;171 const stream = try p.parse();
172 return stream.docs[0];
173}
174
175pub const Parser = struct {
176 alloc: *std.mem.Allocator,
177 tokens: TokenList,
178 lines: Array,
179 index: usize,
180
181 pub fn parse(self: *Parser) !Stream {
182 const item = try parse_item(self, null);
183 return item.stream;
161 }184 }
162185
163 u.assert(all_events.items.len == 1, "failure parsing zig.mod. please report an issue at https://github.com/nektro/zigmod/issues/new that contains the text of your zig.mod.", .{});186 fn next(self: *Parser) ?c.yaml_event_t {
164 return all_events.items[0].document;187 if (self.index >= self.tokens.len) {
165}188 return null;
189 }
190 defer self.index += 1;
191 return self.tokens[self.index];
192 }
193};
166194
167fn get_event_string(event: c.yaml_event_t, lines: Array) []const u8 {195pub const Error =
168 const sm = event.start_mark;196 std.mem.Allocator.Error ||
169 const em = event.end_mark;197 error{YamlUnexpectedToken};
170 return lines[sm.line][sm.column..em.column];198
199fn parse_item(p: *Parser, start: ?c.yaml_event_t) Error!Item {
200 const tok = start orelse p.next();
201 return switch (tok.?.type) {
202 .YAML_STREAM_START_EVENT => Item{ .stream = try parse_stream(p) },
203 .YAML_DOCUMENT_START_EVENT => Item{ .document = try parse_document(p) },
204 .YAML_MAPPING_START_EVENT => Item{ .mapping = try parse_mapping(p) },
205 .YAML_SEQUENCE_START_EVENT => Item{ .sequence = try parse_sequence(p) },
206 .YAML_SCALAR_EVENT => Item{ .string = get_event_string(tok.?, p.lines) },
207 else => unreachable,
208 };
171}209}
172210
173fn condense_event_list(list: *std.ArrayList(Item), lines: Array) !void {211fn parse_stream(p: *Parser) Error!Stream {
174 var i: usize = 0;212 const res = &std.ArrayList(Document).init(p.alloc);
175 var new_list = std.ArrayList(Item).init(list.allocator);213 defer res.deinit();
176214
177 while (i < list.items.len) : (i += 1) {215 while (true) {
178 if (try condense_event_list_key(list.items, i, &new_list, lines)) |len| {216 const tok = p.next();
179 i += len;217 if (tok.?.type == .YAML_STREAM_END_EVENT) {
180 continue;218 return Stream{ .docs = res.toOwnedSlice() };
181 }
182 if (try condense_event_list_mapping(list.items, i, &new_list, lines)) |len| {
183 i += len;
184 continue;
185 }
186 if (try condense_event_list_sequence(list.items, i, &new_list, lines)) |len| {
187 i += len;
188 continue;
189 }219 }
190 if (try condense_event_list_document(list.items, i, &new_list, lines)) |len| {220 if (tok.?.type != .YAML_DOCUMENT_START_EVENT) {
191 i += len;221 return error.YamlUnexpectedToken;
192 continue;
193 }222 }
194 try new_list.append(list.items[i]);223 const item = try parse_item(p, tok);
224 try res.append(item.document);
195 }225 }
196
197 list.deinit();
198 list.* = new_list;
199}226}
200227
201fn condense_event_list_key(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {228fn parse_document(p: *Parser) Error!Document {
202 if (at >= from.len - 1) {229 const tok = p.next();
203 return null;230 if (tok.?.type != .YAML_MAPPING_START_EVENT) {
231 return error.YamlUnexpectedToken;
204 }232 }
205 const t = from[at];233 const item = try parse_item(p, tok);
206 const n = from[at + 1];
207234
208 if (!(t == .event and @enumToInt(t.event.type) == c.YAML_SCALAR_EVENT)) {235 if (p.next().?.type != .YAML_DOCUMENT_END_EVENT) {
209 return null;236 return error.YamlUnexpectedToken;
210 }
211 if (n == .event and @enumToInt(n.event.type) == c.YAML_SCALAR_EVENT) {
212 try to.append(Item{
213 .kv = Key{
214 .key = get_event_string(t.event, lines),
215 .value = Value{ .string = get_event_string(n.event, lines) },
216 },
217 });
218 return 0 + 2 - 1;
219 }237 }
220 if (n == .sequence) {238 return Document{ .mapping = item.mapping };
221 try to.append(Item{
222 .kv = Key{
223 .key = get_event_string(t.event, lines),
224 .value = Value{ .sequence = n.sequence },
225 },
226 });
227 return 0 + 2 - 1;
228 }
229 if (n == .mapping) {
230 try to.append(Item{
231 .kv = Key{
232 .key = get_event_string(t.event, lines),
233 .value = Value{ .mapping = n.mapping },
234 },
235 });
236 return 0 + 2 - 1;
237 }
238 return null;
239}239}
240240
241fn condense_event_list_mapping(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {241fn parse_mapping(p: *Parser) Error!Mapping {
242 if (!(from[at] == .event and @enumToInt(from[at].event.type) == c.YAML_MAPPING_START_EVENT)) {242 const res = &std.ArrayList(Key).init(p.alloc);
243 return null;243 defer res.deinit();
244 }
245 var i: usize = 1;
246 while (true) : (i += 1) {
247 const ele = from[at + i];
248 if (ele == .event and @enumToInt(ele.event.type) == c.YAML_MAPPING_END_EVENT) {
249 break;
250 }
251 if (ele == .event) {
252 return null;
253 }
254 }
255244
256 const keys = &std.ArrayList(Key).init(to.allocator);245 while (true) {
257 for (from[at + 1 .. at + i]) |item| {246 const tok = p.next();
258 switch (item) {247 if (tok.?.type == .YAML_MAPPING_END_EVENT) {
259 .kv => {248 return Mapping{ .items = res.toOwnedSlice() };
260 try keys.append(item.kv);
261 },
262 else => unreachable,
263 }249 }
250 if (tok.?.type != .YAML_SCALAR_EVENT) {
251 return error.YamlUnexpectedToken;
252 }
253 try res.append(Key{
254 .key = get_event_string(tok.?, p.lines),
255 .value = try parse_value(p),
256 });
264 }257 }
258}
265259
266 try to.append(Item{260fn parse_value(p: *Parser) Error!Value {
267 .mapping = Mapping{ .items = keys.items },261 const item = try parse_item(p, null);
268 });262 return switch (item) {
269 return 0 + i;263 .mapping => |x| Value{ .mapping = x },
264 .sequence => |x| Value{ .sequence = x },
265 .string => |x| Value{ .string = x },
266 else => unreachable,
267 };
270}268}
271269
272fn condense_event_list_sequence(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {270fn parse_sequence(p: *Parser) Error!Sequence {
273 if (!(from[at] == .event and @enumToInt(from[at].event.type) == c.YAML_SEQUENCE_START_EVENT)) {271 const res = &std.ArrayList(Item).init(p.alloc);
274 return null;272 defer res.deinit();
275 }
276 var i: usize = 1;
277 while (true) : (i += 1) {
278 const ele = from[at + i];
279 if (ele == .event) {
280 if (@enumToInt(ele.event.type) == c.YAML_SEQUENCE_END_EVENT) {
281 break;
282 }
283 if (@enumToInt(ele.event.type) == c.YAML_SCALAR_EVENT) {
284 continue;
285 }
286 return null;
287 }
288 }
289273
290 const result = &std.ArrayList(Item).init(to.allocator);274 while (true) {
291 for (from[at + 1 .. at + i]) |item| {275 const tok = p.next();
292 try result.append(switch (item) {276 if (tok.?.type == .YAML_SEQUENCE_END_EVENT) {
293 .mapping => item,277 return res.toOwnedSlice();
294 .event => Item{ .string = get_event_string(item.event, lines) },278 }
295 else => unreachable,279 try res.append(try parse_item(p, tok));
296 });
297 }280 }
298 try to.append(Item{
299 .sequence = result.items,
300 });
301 return 0 + i;
302}281}
303282
304fn condense_event_list_document(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {283fn get_event_string(event: c.yaml_event_t, lines: Array) []const u8 {
305 if (from.len < at + 4) {284 const sm = event.start_mark;
306 return null;285 const em = event.end_mark;
307 }286 return lines[sm.line][sm.column..em.column];
308 if (!(from[at] == .event and @enumToInt(from[at].event.type) == c.YAML_STREAM_START_EVENT)) {
309 return null;
310 }
311 if (!(from[at + 1] == .event and @enumToInt(from[at + 1].event.type) == c.YAML_DOCUMENT_START_EVENT)) {
312 return null;
313 }
314 if (!(from[at + 2] == .mapping)) {
315 return null;
316 }
317 if (!(from[at + 3] == .event and @enumToInt(from[at + 3].event.type) == c.YAML_DOCUMENT_END_EVENT)) {
318 return null;
319 }
320 if (!(from[at + 4] == .event and @enumToInt(from[at + 4].event.type) == c.YAML_STREAM_END_EVENT)) {
321 return null;
322 }
323 try to.append(Item{
324 .document = Document{
325 .mapping = from[at + 2].mapping,
326 },
327 });
328 return 0 + 5 - 1;
329}287}