authorgravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2024-06-04 12:33:58 -07:00
committergravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2024-06-04 12:33:58 -07:00
logf4dcf2bc324672fb2bdce20c634d9c5b6d064cc5
tree619b3059ef89b96b94bf3ab878fc7d36ae709881
parent43fbfb46b75705d2cb37c1a806450fe5647fa26e

move core dod plumbing to intrusive_parser.zig

2 files changed, 226 insertions(+), 146 deletions(-)

intrusive_parser.zig created+194
......@@ -0,0 +1,194 @@
1const std = @import("std");
2const string = []const u8;
3const extras = @import("extras");
4const tracer = @import("tracer");
5
6pub const Parser = struct {
7 any: std.io.AnyReader,
8 allocator: std.mem.Allocator,
9 temp: std.ArrayListUnmanaged(u8) = .{},
10 idx: usize = 0,
11 end: bool = false,
12 line: usize = 1,
13 col: usize = 1,
14 data: std.ArrayListUnmanaged(u8) = .{},
15 strings_map: std.StringArrayHashMapUnmanaged(usize) = .{},
16 string_tag: u8,
17
18 pub fn init(allocator: std.mem.Allocator, any: std.io.AnyReader, string_tag: u8) Parser {
19 return .{
20 .any = any,
21 .allocator = allocator,
22 .string_tag = string_tag,
23 };
24 }
25
26 pub fn deinit(p: *Parser) void {
27 p.temp.deinit(p.allocator);
28 p.data.deinit(p.allocator);
29 p.strings_map.deinit(p.allocator);
30 }
31
32 pub fn avail(p: *Parser) usize {
33 return p.temp.items.len - p.idx;
34 }
35
36 pub fn slice(p: *Parser) []const u8 {
37 return p.temp.items[p.idx..];
38 }
39
40 pub fn eat(p: *Parser, comptime test_s: string) !?void {
41 if (test_s.len == 1) {
42 _ = try p.eatByte(test_s[0]);
43 return;
44 }
45 try p.peekAmt(test_s.len) orelse return null;
46 if (std.mem.eql(u8, p.slice()[0..test_s.len], test_s)) {
47 p.idx += test_s.len;
48 return;
49 }
50 return null;
51 }
52
53 fn peekAmt(p: *Parser, amt: usize) !?void {
54 if (p.avail() >= amt) return;
55 const buf_size = std.mem.page_size;
56 const diff_amt = amt - p.avail();
57 std.debug.assert(diff_amt <= buf_size);
58 var buf: [buf_size]u8 = undefined;
59 const len = try p.any.readAll(&buf);
60 if (len == 0) p.end = true;
61 if (len == 0) return null;
62 try p.temp.appendSlice(p.allocator, buf[0..len]);
63 if (amt > len) return null;
64 }
65
66 pub fn eatByte(p: *Parser, test_c: u8) !?u8 {
67 const t = tracer.trace(@src(), " '{c}'", .{test_c});
68 defer t.end();
69
70 try p.peekAmt(1) orelse return null;
71 if (p.slice()[0] == test_c) {
72 p.idx += 1;
73 return test_c;
74 }
75 return null;
76 }
77
78 pub fn eatRange(p: *Parser, comptime from: u8, comptime to: u8) !?u8 {
79 const t = tracer.trace(@src(), " ({d},{d})", .{ from, to });
80 defer t.end();
81
82 try p.peekAmt(1) orelse return null;
83 if (p.slice()[0] >= from and p.slice()[0] <= to) {
84 defer p.idx += 1;
85 return p.slice()[0];
86 }
87 return null;
88 }
89
90 pub fn eatAnyScalar(p: *Parser, test_s: string) !?u8 {
91 const t = tracer.trace(@src(), " ({s})", .{test_s});
92 defer t.end();
93
94 std.debug.assert(extras.matchesAll(u8, test_s, std.ascii.isASCII));
95 try p.peekAmt(1) orelse return null;
96 if (std.mem.indexOfScalar(u8, test_s, p.slice()[0])) |idx| {
97 p.idx += 1;
98 return test_s[idx];
99 }
100 return null;
101 }
102
103 pub fn shift(p: *Parser) !u21 {
104 try p.peekAmt(1) orelse return error.EndOfStream;
105 const len = std.unicode.utf8ByteSequenceLength(p.slice()[0]) catch return error.MalformedJson;
106 try p.peekAmt(len) orelse return error.EndOfStream;
107 defer p.idx += len;
108 return std.unicode.utf8Decode(p.slice()[0..len]) catch return error.MalformedJson;
109 }
110
111 pub fn shiftBytesN(p: *Parser, comptime n: usize) ![n]u8 {
112 try p.peekAmt(n) orelse return error.EndOfStream;
113 defer p.idx += n;
114 return p.slice()[0..n].*;
115 }
116
117 // tag(u8) + len(u32) + bytes(N)
118 pub fn addStr(p: *Parser, alloc: std.mem.Allocator, str: string) !usize {
119 const t = tracer.trace(@src(), "({d})", .{str.len});
120 defer t.end();
121
122 const adapter: AdapterStr = .{ .p = p };
123 const res = try p.strings_map.getOrPutAdapted(alloc, str, adapter);
124 if (res.found_existing) return res.value_ptr.*;
125 errdefer p.strings_map.orderedRemoveAt(res.index);
126 const r = p.data.items.len;
127 const l = str.len;
128 try p.data.ensureUnusedCapacity(alloc, 1 + 4 + l);
129 p.data.appendAssumeCapacity(p.string_tag);
130 p.data.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
131 p.data.appendSliceAssumeCapacity(str);
132 res.value_ptr.* = r;
133 return r;
134 }
135
136 const AdapterStr = struct {
137 p: *const Parser,
138
139 pub fn hash(ctx: @This(), a: string) u32 {
140 _ = ctx;
141 var hasher = std.hash.Wyhash.init(0);
142 hasher.update(a);
143 return @truncate(hasher.final());
144 }
145
146 pub fn eql(ctx: @This(), a: string, _: string, b_index: usize) bool {
147 const i = ctx.p.strings_map.values()[b_index];
148 std.debug.assert(ctx.p.data.items[i] == ctx.p.string_tag);
149 const l: u32 = @bitCast(ctx.p.data.items[i..][1..][0..4].*);
150 const b = ctx.p.data.items[i..][1..][4..][0..l];
151 return std.mem.eql(u8, a, b);
152 }
153 };
154};
155
156pub fn Mixin(comptime T: type) type {
157 return struct {
158 pub fn avail(pp: *T) usize {
159 return pp.parser.avail();
160 }
161
162 pub fn slice(pp: *T) []const u8 {
163 return pp.parser.slice();
164 }
165
166 pub fn eat(pp: *T, comptime test_s: string) !?void {
167 return pp.parser.eat(test_s);
168 }
169
170 fn peekAmt(pp: *T, amt: usize) !?void {
171 return pp.parser.peekAmt(amt);
172 }
173
174 pub fn eatByte(pp: *T, test_c: u8) !?u8 {
175 return pp.parser.eatByte(test_c);
176 }
177
178 pub fn eatRange(pp: *T, comptime from: u8, comptime to: u8) !?u8 {
179 return pp.parser.eatRange(from, to);
180 }
181
182 pub fn eatAnyScalar(pp: *T, test_s: string) !?u8 {
183 return pp.parser.eatAnyScalar(test_s);
184 }
185
186 pub fn shift(pp: *T) !u21 {
187 return pp.parser.shift();
188 }
189
190 pub fn shiftBytesN(pp: *T, comptime n: usize) ![n]u8 {
191 return pp.parser.shiftBytesN(n);
192 }
193 };
194}
json.zig+32-146
......@@ -2,6 +2,7 @@ const std = @import("std");
22const string = []const u8;
33const extras = @import("extras");
44const tracer = @import("tracer");
5const intrusive_parser = @import("./intrusive_parser.zig");
56
67const Error = error{ OutOfMemory, EndOfStream, MalformedJson };
78const ObjectHashMap = std.AutoArrayHashMapUnmanaged(StringIndex, ValueIndex);
......@@ -13,17 +14,15 @@ pub fn parse(alloc: std.mem.Allocator, path: string, inreader: anytype, options:
1314 _ = path;
1415
1516 var p = Parser.init(alloc, inreader.any(), options);
16 defer p.temp.deinit(alloc);
17 defer p.strings_map.deinit(alloc);
17 defer p.parser.deinit();
1818 defer p.numbers_map.deinit(alloc);
19 errdefer p.extras.deinit(alloc);
2019
2120 comptime std.debug.assert(@intFromEnum(Value.zero) == 0);
22 try p.extras.ensureUnusedCapacity(alloc, 4096);
23 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.zero));
24 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.null));
25 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.true));
26 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.false));
21 try p.parser.data.ensureUnusedCapacity(alloc, 4096);
22 p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.zero));
23 p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.null));
24 p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.true));
25 p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.false));
2726 _ = try p.addStr(alloc, "");
2827 std.debug.assert(try p.addArray(alloc, &.{}) == .empty_array);
2928 std.debug.assert(try p.addObject(alloc, &ObjectHashMap{}) == .empty_object);
......@@ -31,7 +30,7 @@ pub fn parse(alloc: std.mem.Allocator, path: string, inreader: anytype, options:
3130 const root_err: (@TypeOf(inreader).Error || Error)!ValueIndex = @errorCast(parseElement(alloc, &p));
3231 const root = try root_err;
3332 if (p.avail() > 0) return error.MalformedJson;
34 const data = try p.extras.toOwnedSlice(alloc);
33 const data = try p.parser.data.toOwnedSlice(alloc);
3534
3635 return .{
3736 .root = root,
......@@ -162,7 +161,7 @@ fn parseString(alloc: std.mem.Allocator, p: *Parser) anyerror!?StringIndex {
162161 if (c != '\\') {
163162 if (c < 0x20) return error.MalformedJson;
164163 const l = std.unicode.utf8CodepointSequenceLength(c) catch unreachable;
165 const b = p.temp.items[p.idx - l ..][0..l];
164 const b = p.parser.temp.items[p.parser.idx - l ..][0..l];
166165 try characters.appendSlice(b);
167166 continue;
168167 }
......@@ -250,15 +249,7 @@ fn parseWs(p: *Parser) !void {
250249}
251250
252251const Parser = struct {
253 any: std.io.AnyReader,
254 arena: std.mem.Allocator,
255 temp: std.ArrayListUnmanaged(u8) = .{},
256 idx: usize = 0,
257 end: bool = false,
258 line: usize = 1,
259 col: usize = 1,
260 extras: std.ArrayListUnmanaged(u8) = .{},
261 strings_map: std.StringArrayHashMapUnmanaged(StringIndex) = .{},
252 parser: intrusive_parser.Parser,
262253 numbers_map: std.StringArrayHashMapUnmanaged(NumberIndex) = .{},
263254 depth: u16 = 0,
264255
......@@ -267,8 +258,7 @@ const Parser = struct {
267258
268259 pub fn init(allocator: std.mem.Allocator, any: std.io.AnyReader, options: Options) Parser {
269260 return .{
270 .any = any,
271 .arena = allocator,
261 .parser = intrusive_parser.Parser.init(allocator, any, @intFromEnum(Value.Tag.string)),
272262 .support_trailing_commas = options.support_trailing_commas,
273263 .maximum_depth = options.maximum_depth,
274264 };
......@@ -279,104 +269,21 @@ const Parser = struct {
279269 maximum_depth: u16,
280270 };
281271
282 pub fn avail(p: *Parser) usize {
283 return p.temp.items.len - p.idx;
284 }
285
286 pub fn slice(p: *Parser) []const u8 {
287 return p.temp.items[p.idx..];
288 }
289
290 pub fn eat(p: *Parser, comptime test_s: string) !?void {
291 if (test_s.len == 1) {
292 _ = try p.eatByte(test_s[0]);
293 return;
294 }
295 try p.peekAmt(test_s.len) orelse return null;
296 if (std.mem.eql(u8, p.slice()[0..test_s.len], test_s)) {
297 p.idx += test_s.len;
298 return;
299 }
300 return null;
301 }
302
303 fn peekAmt(p: *Parser, amt: usize) !?void {
304 if (p.avail() >= amt) return;
305 const buf_size = std.mem.page_size;
306 const diff_amt = amt - p.avail();
307 std.debug.assert(diff_amt <= buf_size);
308 var buf: [buf_size]u8 = undefined;
309 const len = try p.any.readAll(&buf);
310 if (len == 0) p.end = true;
311 if (len == 0) return null;
312 try p.temp.appendSlice(p.arena, buf[0..len]);
313 if (amt > len) return null;
314 }
315
316 pub fn eatByte(p: *Parser, test_c: u8) !?u8 {
317 const t = tracer.trace(@src(), " '{c}'", .{test_c});
318 defer t.end();
319
320 try p.peekAmt(1) orelse return null;
321 if (p.slice()[0] == test_c) {
322 p.idx += 1;
323 return test_c;
324 }
325 return null;
326 }
327
328 pub fn eatRange(p: *Parser, comptime from: u8, comptime to: u8) !?u8 {
329 const t = tracer.trace(@src(), " ({d},{d})", .{ from, to });
330 defer t.end();
331
332 try p.peekAmt(1) orelse return null;
333 if (p.slice()[0] >= from and p.slice()[0] <= to) {
334 defer p.idx += 1;
335 return p.slice()[0];
336 }
337 return null;
338 }
339
340 pub fn eatAnyScalar(p: *Parser, test_s: string) !?u8 {
341 const t = tracer.trace(@src(), " ({s})", .{test_s});
342 defer t.end();
343
344 std.debug.assert(extras.matchesAll(u8, test_s, std.ascii.isASCII));
345 try p.peekAmt(1) orelse return null;
346 if (std.mem.indexOfScalar(u8, test_s, p.slice()[0])) |idx| {
347 p.idx += 1;
348 return test_s[idx];
349 }
350 return null;
351 }
352
353 pub fn shift(p: *Parser) !u21 {
354 try p.peekAmt(1) orelse return error.EndOfStream;
355 const len = std.unicode.utf8ByteSequenceLength(p.slice()[0]) catch return error.MalformedJson;
356 try p.peekAmt(len) orelse return error.EndOfStream;
357 defer p.idx += len;
358 return std.unicode.utf8Decode(p.slice()[0..len]) catch return error.MalformedJson;
359 }
360
361 pub fn shiftBytesN(p: *Parser, comptime n: usize) ![n]u8 {
362 try p.peekAmt(n) orelse return error.EndOfStream;
363 defer p.idx += n;
364 return p.slice()[0..n].*;
365 }
272 pub usingnamespace intrusive_parser.Mixin(@This());
366273
367274 // tag(u8) + len(u32) + member_keys(N * u32) + member_values(N * u32)
368275 pub fn addObject(p: *Parser, alloc: std.mem.Allocator, members: *const ObjectHashMap) !ValueIndex {
369276 const t = tracer.trace(@src(), "({d})", .{members.entries.len});
370277 defer t.end();
371278
372 const r = p.extras.items.len;
279 const r = p.parser.data.items.len;
373280 const l = members.entries.len;
374281 if (l > std.math.maxInt(u32)) return error.MalformedJson;
375 try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4 * 2));
376 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.object));
377 p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
378 p.extras.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.keys()));
379 p.extras.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.values()));
282 try p.parser.data.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4 * 2));
283 p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.object));
284 p.parser.data.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
285 p.parser.data.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.keys()));
286 p.parser.data.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.values()));
380287 return @enumFromInt(r);
381288 }
382289
......@@ -385,33 +292,19 @@ const Parser = struct {
385292 const t = tracer.trace(@src(), "({d})", .{items.len});
386293 defer t.end();
387294
388 const r = p.extras.items.len;
295 const r = p.parser.data.items.len;
389296 const l = items.len;
390297 if (l > std.math.maxInt(u32)) return error.MalformedJson;
391 try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4));
392 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.array));
393 p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
394 p.extras.appendSliceAssumeCapacity(std.mem.sliceAsBytes(items));
298 try p.parser.data.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4));
299 p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.array));
300 p.parser.data.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
301 p.parser.data.appendSliceAssumeCapacity(std.mem.sliceAsBytes(items));
395302 return @enumFromInt(r);
396303 }
397304
398305 // tag(u8) + len(u32) + bytes(N)
399306 pub fn addStr(p: *Parser, alloc: std.mem.Allocator, str: string) !StringIndex {
400 const t = tracer.trace(@src(), "({d})", .{str.len});
401 defer t.end();
402
403 const adapter: Adapter = .{ .p = p };
404 const res = try p.strings_map.getOrPutAdapted(alloc, str, adapter);
405 if (res.found_existing) return res.value_ptr.*;
406 errdefer p.strings_map.orderedRemoveAt(res.index);
407 const r = p.extras.items.len;
408 const l = str.len;
409 try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + l);
410 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.string));
411 p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
412 p.extras.appendSliceAssumeCapacity(str);
413 res.value_ptr.* = @enumFromInt(r);
414 return @enumFromInt(r);
307 return @enumFromInt(try p.parser.addStr(alloc, str));
415308 }
416309
417310 const Adapter = struct {
......@@ -439,12 +332,12 @@ const Parser = struct {
439332 const res = try p.numbers_map.getOrPutAdapted(alloc, v, adapter);
440333 if (res.found_existing) return @enumFromInt(@intFromEnum(res.value_ptr.*));
441334 errdefer p.numbers_map.orderedRemoveAt(res.index);
442 const r = p.extras.items.len;
335 const r = p.parser.data.items.len;
443336 const l = v.len;
444 try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + l);
445 p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.number));
446 p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
447 p.extras.appendSliceAssumeCapacity(v);
337 try p.parser.data.ensureUnusedCapacity(alloc, 1 + 4 + l);
338 p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.number));
339 p.parser.data.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l))));
340 p.parser.data.appendSliceAssumeCapacity(v);
448341 res.value_ptr.* = @enumFromInt(r);
449342 return @enumFromInt(r);
450343 }
......@@ -462,19 +355,12 @@ const Parser = struct {
462355 pub fn eql(ctx: @This(), a: string, _: string, b_index: usize) bool {
463356 const sidx = ctx.p.numbers_map.values()[b_index];
464357 const i: u32 = @intFromEnum(sidx);
465 std.debug.assert(@as(Value.Tag, @enumFromInt(ctx.p.extras.items[i])) == .number);
466 const l: u32 = @bitCast(ctx.p.extras.items[i..][1..][0..4].*);
467 const b = ctx.p.extras.items[i..][1..][4..][0..l];
358 std.debug.assert(@as(Value.Tag, @enumFromInt(ctx.p.parser.data.items[i])) == .number);
359 const l: u32 = @bitCast(ctx.p.parser.data.items[i..][1..][0..4].*);
360 const b = ctx.p.parser.data.items[i..][1..][4..][0..l];
468361 return std.mem.eql(u8, a, b);
469362 }
470363 };
471
472 pub fn getStr(p: *const Parser, sidx: StringIndex) string {
473 const i: u32 = @intFromEnum(sidx);
474 std.debug.assert(@as(Value.Tag, @enumFromInt(p.extras.items[i])) == .string);
475 const l: u32 = @bitCast(p.extras.items[i..][1..][0..4].*);
476 return p.extras.items[i..][1..][4..][0..l];
477 }
478364};
479365
480366pub threadlocal var doc: ?*const Document = null;