authorgravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2021-06-07 05:34:17 -07:00
committergravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2021-06-07 05:34:17 -07:00
logeaf70895de1efa8d01d58268919c4f7737554824
tree2471983bb767f3af77f7143e32b69537f99eb075
parent2076b1bd3dfc651cbe4bad90aac9e8a59454ad0a

lib/yaml- rework to be O(n) and much more memory efficient

1 files changed, 110 insertions(+), 152 deletions(-)

src/util/yaml.zig+110-152
......@@ -1,4 +1,5 @@
11const std = @import("std");
2const yaml = @import("yaml");
23
34const c = @cImport({
45 @cInclude("yaml.h");
......@@ -10,6 +11,10 @@ const u = @import("./index.zig");
1011
1112const Array = []const []const u8;
1213
14pub const Stream = struct {
15 docs: []const Document,
16};
17
1318pub const Document = struct {
1419 mapping: Mapping,
1520};
......@@ -21,11 +26,15 @@ pub const Item = union(enum) {
2126 sequence: Sequence,
2227 document: Document,
2328 string: []const u8,
29 stream: Stream,
2430
2531 pub fn format(self: Item, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) @TypeOf(writer).Error!void {
2632 try writer.writeAll("Item{");
2733 switch (self) {
28 .event, .kv, .document => {
34 .event => {
35 try std.fmt.format(writer, "{s}", .{@tagName(self.event.type)});
36 },
37 .kv, .document => {
2938 unreachable;
3039 },
3140 .mapping => {
......@@ -121,6 +130,8 @@ pub const Mapping = struct {
121130 }
122131};
123132
133pub const TokenList = []const c.yaml_event_t;
134
124135//
125136//
126137
......@@ -132,7 +143,7 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {
132143
133144 _ = c.yaml_parser_set_input_string(&parser, input.ptr, input.len);
134145
135 const all_events = &std.ArrayList(Item).init(alloc);
146 const all_events = &std.ArrayList(c.yaml_event_t).init(alloc);
136147 var event: c.yaml_event_t = undefined;
137148 while (true) {
138149 const p = c.yaml_parser_parse(&parser, &event);
......@@ -141,7 +152,7 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {
141152 }
142153
143154 const et = @enumToInt(event.type);
144 try all_events.append(.{ .event = event });
155 try all_events.append(event);
145156 c.yaml_event_delete(&event);
146157
147158 if (et == c.YAML_STREAM_END_EVENT) {
......@@ -151,179 +162,126 @@ pub fn parse(alloc: *std.mem.Allocator, input: []const u8) !Document {
151162
152163 c.yaml_parser_delete(&parser);
153164
154 var l: usize = all_events.items.len;
155 while (true) {
156 try condense_event_list(all_events, lines);
157 if (l == all_events.items.len) {
158 break;
159 }
160 l = all_events.items.len;
165 const p = &Parser{
166 .alloc = alloc,
167 .tokens = all_events.items,
168 .lines = lines,
169 .index = 0,
170 };
171 const stream = try p.parse();
172 return stream.docs[0];
173}
174
175pub const Parser = struct {
176 alloc: *std.mem.Allocator,
177 tokens: TokenList,
178 lines: Array,
179 index: usize,
180
181 pub fn parse(self: *Parser) !Stream {
182 const item = try parse_item(self, null);
183 return item.stream;
161184 }
162185
163 u.assert(all_events.items.len == 1, "failure parsing zig.mod. please report an issue at https://github.com/nektro/zigmod/issues/new that contains the text of your zig.mod.", .{});
164 return all_events.items[0].document;
165}
186 fn next(self: *Parser) ?c.yaml_event_t {
187 if (self.index >= self.tokens.len) {
188 return null;
189 }
190 defer self.index += 1;
191 return self.tokens[self.index];
192 }
193};
166194
167fn get_event_string(event: c.yaml_event_t, lines: Array) []const u8 {
168 const sm = event.start_mark;
169 const em = event.end_mark;
170 return lines[sm.line][sm.column..em.column];
195pub const Error =
196 std.mem.Allocator.Error ||
197 error{YamlUnexpectedToken};
198
199fn parse_item(p: *Parser, start: ?c.yaml_event_t) Error!Item {
200 const tok = start orelse p.next();
201 return switch (tok.?.type) {
202 .YAML_STREAM_START_EVENT => Item{ .stream = try parse_stream(p) },
203 .YAML_DOCUMENT_START_EVENT => Item{ .document = try parse_document(p) },
204 .YAML_MAPPING_START_EVENT => Item{ .mapping = try parse_mapping(p) },
205 .YAML_SEQUENCE_START_EVENT => Item{ .sequence = try parse_sequence(p) },
206 .YAML_SCALAR_EVENT => Item{ .string = get_event_string(tok.?, p.lines) },
207 else => unreachable,
208 };
171209}
172210
173fn condense_event_list(list: *std.ArrayList(Item), lines: Array) !void {
174 var i: usize = 0;
175 var new_list = std.ArrayList(Item).init(list.allocator);
211fn parse_stream(p: *Parser) Error!Stream {
212 const res = &std.ArrayList(Document).init(p.alloc);
213 defer res.deinit();
176214
177 while (i < list.items.len) : (i += 1) {
178 if (try condense_event_list_key(list.items, i, &new_list, lines)) |len| {
179 i += len;
180 continue;
181 }
182 if (try condense_event_list_mapping(list.items, i, &new_list, lines)) |len| {
183 i += len;
184 continue;
185 }
186 if (try condense_event_list_sequence(list.items, i, &new_list, lines)) |len| {
187 i += len;
188 continue;
215 while (true) {
216 const tok = p.next();
217 if (tok.?.type == .YAML_STREAM_END_EVENT) {
218 return Stream{ .docs = res.toOwnedSlice() };
189219 }
190 if (try condense_event_list_document(list.items, i, &new_list, lines)) |len| {
191 i += len;
192 continue;
220 if (tok.?.type != .YAML_DOCUMENT_START_EVENT) {
221 return error.YamlUnexpectedToken;
193222 }
194 try new_list.append(list.items[i]);
223 const item = try parse_item(p, tok);
224 try res.append(item.document);
195225 }
196
197 list.deinit();
198 list.* = new_list;
199226}
200227
201fn condense_event_list_key(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {
202 if (at >= from.len - 1) {
203 return null;
228fn parse_document(p: *Parser) Error!Document {
229 const tok = p.next();
230 if (tok.?.type != .YAML_MAPPING_START_EVENT) {
231 return error.YamlUnexpectedToken;
204232 }
205 const t = from[at];
206 const n = from[at + 1];
233 const item = try parse_item(p, tok);
207234
208 if (!(t == .event and @enumToInt(t.event.type) == c.YAML_SCALAR_EVENT)) {
209 return null;
210 }
211 if (n == .event and @enumToInt(n.event.type) == c.YAML_SCALAR_EVENT) {
212 try to.append(Item{
213 .kv = Key{
214 .key = get_event_string(t.event, lines),
215 .value = Value{ .string = get_event_string(n.event, lines) },
216 },
217 });
218 return 0 + 2 - 1;
235 if (p.next().?.type != .YAML_DOCUMENT_END_EVENT) {
236 return error.YamlUnexpectedToken;
219237 }
220 if (n == .sequence) {
221 try to.append(Item{
222 .kv = Key{
223 .key = get_event_string(t.event, lines),
224 .value = Value{ .sequence = n.sequence },
225 },
226 });
227 return 0 + 2 - 1;
228 }
229 if (n == .mapping) {
230 try to.append(Item{
231 .kv = Key{
232 .key = get_event_string(t.event, lines),
233 .value = Value{ .mapping = n.mapping },
234 },
235 });
236 return 0 + 2 - 1;
237 }
238 return null;
238 return Document{ .mapping = item.mapping };
239239}
240240
241fn condense_event_list_mapping(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {
242 if (!(from[at] == .event and @enumToInt(from[at].event.type) == c.YAML_MAPPING_START_EVENT)) {
243 return null;
244 }
245 var i: usize = 1;
246 while (true) : (i += 1) {
247 const ele = from[at + i];
248 if (ele == .event and @enumToInt(ele.event.type) == c.YAML_MAPPING_END_EVENT) {
249 break;
250 }
251 if (ele == .event) {
252 return null;
253 }
254 }
241fn parse_mapping(p: *Parser) Error!Mapping {
242 const res = &std.ArrayList(Key).init(p.alloc);
243 defer res.deinit();
255244
256 const keys = &std.ArrayList(Key).init(to.allocator);
257 for (from[at + 1 .. at + i]) |item| {
258 switch (item) {
259 .kv => {
260 try keys.append(item.kv);
261 },
262 else => unreachable,
245 while (true) {
246 const tok = p.next();
247 if (tok.?.type == .YAML_MAPPING_END_EVENT) {
248 return Mapping{ .items = res.toOwnedSlice() };
263249 }
250 if (tok.?.type != .YAML_SCALAR_EVENT) {
251 return error.YamlUnexpectedToken;
252 }
253 try res.append(Key{
254 .key = get_event_string(tok.?, p.lines),
255 .value = try parse_value(p),
256 });
264257 }
258}
265259
266 try to.append(Item{
267 .mapping = Mapping{ .items = keys.items },
268 });
269 return 0 + i;
260fn parse_value(p: *Parser) Error!Value {
261 const item = try parse_item(p, null);
262 return switch (item) {
263 .mapping => |x| Value{ .mapping = x },
264 .sequence => |x| Value{ .sequence = x },
265 .string => |x| Value{ .string = x },
266 else => unreachable,
267 };
270268}
271269
272fn condense_event_list_sequence(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {
273 if (!(from[at] == .event and @enumToInt(from[at].event.type) == c.YAML_SEQUENCE_START_EVENT)) {
274 return null;
275 }
276 var i: usize = 1;
277 while (true) : (i += 1) {
278 const ele = from[at + i];
279 if (ele == .event) {
280 if (@enumToInt(ele.event.type) == c.YAML_SEQUENCE_END_EVENT) {
281 break;
282 }
283 if (@enumToInt(ele.event.type) == c.YAML_SCALAR_EVENT) {
284 continue;
285 }
286 return null;
287 }
288 }
270fn parse_sequence(p: *Parser) Error!Sequence {
271 const res = &std.ArrayList(Item).init(p.alloc);
272 defer res.deinit();
289273
290 const result = &std.ArrayList(Item).init(to.allocator);
291 for (from[at + 1 .. at + i]) |item| {
292 try result.append(switch (item) {
293 .mapping => item,
294 .event => Item{ .string = get_event_string(item.event, lines) },
295 else => unreachable,
296 });
274 while (true) {
275 const tok = p.next();
276 if (tok.?.type == .YAML_SEQUENCE_END_EVENT) {
277 return res.toOwnedSlice();
278 }
279 try res.append(try parse_item(p, tok));
297280 }
298 try to.append(Item{
299 .sequence = result.items,
300 });
301 return 0 + i;
302281}
303282
304fn condense_event_list_document(from: []Item, at: usize, to: *std.ArrayList(Item), lines: Array) !?usize {
305 if (from.len < at + 4) {
306 return null;
307 }
308 if (!(from[at] == .event and @enumToInt(from[at].event.type) == c.YAML_STREAM_START_EVENT)) {
309 return null;
310 }
311 if (!(from[at + 1] == .event and @enumToInt(from[at + 1].event.type) == c.YAML_DOCUMENT_START_EVENT)) {
312 return null;
313 }
314 if (!(from[at + 2] == .mapping)) {
315 return null;
316 }
317 if (!(from[at + 3] == .event and @enumToInt(from[at + 3].event.type) == c.YAML_DOCUMENT_END_EVENT)) {
318 return null;
319 }
320 if (!(from[at + 4] == .event and @enumToInt(from[at + 4].event.type) == c.YAML_STREAM_END_EVENT)) {
321 return null;
322 }
323 try to.append(Item{
324 .document = Document{
325 .mapping = from[at + 2].mapping,
326 },
327 });
328 return 0 + 5 - 1;
283fn get_event_string(event: c.yaml_event_t, lines: Array) []const u8 {
284 const sm = event.start_mark;
285 const em = event.end_mark;
286 return lines[sm.line][sm.column..em.column];
329287}