| ... | @@ -2,6 +2,7 @@ const std = @import("std"); | ... | @@ -2,6 +2,7 @@ const std = @import("std"); |
| 2 | const string = []const u8; | 2 | const string = []const u8; |
| 3 | const extras = @import("extras"); | 3 | const extras = @import("extras"); |
| 4 | const tracer = @import("tracer"); | 4 | const tracer = @import("tracer"); |
| | 5 | const intrusive_parser = @import("./intrusive_parser.zig"); |
| 5 | | 6 | |
| 6 | const Error = error{ OutOfMemory, EndOfStream, MalformedJson }; | 7 | const Error = error{ OutOfMemory, EndOfStream, MalformedJson }; |
| 7 | const ObjectHashMap = std.AutoArrayHashMapUnmanaged(StringIndex, ValueIndex); | 8 | const ObjectHashMap = std.AutoArrayHashMapUnmanaged(StringIndex, ValueIndex); |
| ... | @@ -13,17 +14,15 @@ pub fn parse(alloc: std.mem.Allocator, path: string, inreader: anytype, options: | ... | @@ -13,17 +14,15 @@ pub fn parse(alloc: std.mem.Allocator, path: string, inreader: anytype, options: |
| 13 | _ = path; | 14 | _ = path; |
| 14 | | 15 | |
| 15 | var p = Parser.init(alloc, inreader.any(), options); | 16 | var p = Parser.init(alloc, inreader.any(), options); |
| 16 | defer p.temp.deinit(alloc); | 17 | defer p.parser.deinit(); |
| 17 | defer p.strings_map.deinit(alloc); | | |
| 18 | defer p.numbers_map.deinit(alloc); | 18 | defer p.numbers_map.deinit(alloc); |
| 19 | errdefer p.extras.deinit(alloc); | | |
| 20 | | 19 | |
| 21 | comptime std.debug.assert(@intFromEnum(Value.zero) == 0); | 20 | comptime std.debug.assert(@intFromEnum(Value.zero) == 0); |
| 22 | try p.extras.ensureUnusedCapacity(alloc, 4096); | 21 | try p.parser.data.ensureUnusedCapacity(alloc, 4096); |
| 23 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.zero)); | 22 | p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.zero)); |
| 24 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.null)); | 23 | p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.null)); |
| 25 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.true)); | 24 | p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.true)); |
| 26 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.false)); | 25 | p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.false)); |
| 27 | _ = try p.addStr(alloc, ""); | 26 | _ = try p.addStr(alloc, ""); |
| 28 | std.debug.assert(try p.addArray(alloc, &.{}) == .empty_array); | 27 | std.debug.assert(try p.addArray(alloc, &.{}) == .empty_array); |
| 29 | std.debug.assert(try p.addObject(alloc, &ObjectHashMap{}) == .empty_object); | 28 | std.debug.assert(try p.addObject(alloc, &ObjectHashMap{}) == .empty_object); |
| ... | @@ -31,7 +30,7 @@ pub fn parse(alloc: std.mem.Allocator, path: string, inreader: anytype, options: | ... | @@ -31,7 +30,7 @@ pub fn parse(alloc: std.mem.Allocator, path: string, inreader: anytype, options: |
| 31 | const root_err: (@TypeOf(inreader).Error || Error)!ValueIndex = @errorCast(parseElement(alloc, &p)); | 30 | const root_err: (@TypeOf(inreader).Error || Error)!ValueIndex = @errorCast(parseElement(alloc, &p)); |
| 32 | const root = try root_err; | 31 | const root = try root_err; |
| 33 | if (p.avail() > 0) return error.MalformedJson; | 32 | if (p.avail() > 0) return error.MalformedJson; |
| 34 | const data = try p.extras.toOwnedSlice(alloc); | 33 | const data = try p.parser.data.toOwnedSlice(alloc); |
| 35 | | 34 | |
| 36 | return .{ | 35 | return .{ |
| 37 | .root = root, | 36 | .root = root, |
| ... | @@ -162,7 +161,7 @@ fn parseString(alloc: std.mem.Allocator, p: *Parser) anyerror!?StringIndex { | ... | @@ -162,7 +161,7 @@ fn parseString(alloc: std.mem.Allocator, p: *Parser) anyerror!?StringIndex { |
| 162 | if (c != '\\') { | 161 | if (c != '\\') { |
| 163 | if (c < 0x20) return error.MalformedJson; | 162 | if (c < 0x20) return error.MalformedJson; |
| 164 | const l = std.unicode.utf8CodepointSequenceLength(c) catch unreachable; | 163 | const l = std.unicode.utf8CodepointSequenceLength(c) catch unreachable; |
| 165 | const b = p.temp.items[p.idx - l ..][0..l]; | 164 | const b = p.parser.temp.items[p.parser.idx - l ..][0..l]; |
| 166 | try characters.appendSlice(b); | 165 | try characters.appendSlice(b); |
| 167 | continue; | 166 | continue; |
| 168 | } | 167 | } |
| ... | @@ -250,15 +249,7 @@ fn parseWs(p: *Parser) !void { | ... | @@ -250,15 +249,7 @@ fn parseWs(p: *Parser) !void { |
| 250 | } | 249 | } |
| 251 | | 250 | |
| 252 | const Parser = struct { | 251 | const Parser = struct { |
| 253 | any: std.io.AnyReader, | 252 | parser: intrusive_parser.Parser, |
| 254 | arena: std.mem.Allocator, | | |
| 255 | temp: std.ArrayListUnmanaged(u8) = .{}, | | |
| 256 | idx: usize = 0, | | |
| 257 | end: bool = false, | | |
| 258 | line: usize = 1, | | |
| 259 | col: usize = 1, | | |
| 260 | extras: std.ArrayListUnmanaged(u8) = .{}, | | |
| 261 | strings_map: std.StringArrayHashMapUnmanaged(StringIndex) = .{}, | | |
| 262 | numbers_map: std.StringArrayHashMapUnmanaged(NumberIndex) = .{}, | 253 | numbers_map: std.StringArrayHashMapUnmanaged(NumberIndex) = .{}, |
| 263 | depth: u16 = 0, | 254 | depth: u16 = 0, |
| 264 | | 255 | |
| ... | @@ -267,8 +258,7 @@ const Parser = struct { | ... | @@ -267,8 +258,7 @@ const Parser = struct { |
| 267 | | 258 | |
| 268 | pub fn init(allocator: std.mem.Allocator, any: std.io.AnyReader, options: Options) Parser { | 259 | pub fn init(allocator: std.mem.Allocator, any: std.io.AnyReader, options: Options) Parser { |
| 269 | return .{ | 260 | return .{ |
| 270 | .any = any, | 261 | .parser = intrusive_parser.Parser.init(allocator, any, @intFromEnum(Value.Tag.string)), |
| 271 | .arena = allocator, | | |
| 272 | .support_trailing_commas = options.support_trailing_commas, | 262 | .support_trailing_commas = options.support_trailing_commas, |
| 273 | .maximum_depth = options.maximum_depth, | 263 | .maximum_depth = options.maximum_depth, |
| 274 | }; | 264 | }; |
| ... | @@ -279,104 +269,21 @@ const Parser = struct { | ... | @@ -279,104 +269,21 @@ const Parser = struct { |
| 279 | maximum_depth: u16, | 269 | maximum_depth: u16, |
| 280 | }; | 270 | }; |
| 281 | | 271 | |
| 282 | pub fn avail(p: *Parser) usize { | 272 | pub usingnamespace intrusive_parser.Mixin(@This()); |
| 283 | return p.temp.items.len - p.idx; | | |
| 284 | } | | |
| 285 | | | |
| 286 | pub fn slice(p: *Parser) []const u8 { | | |
| 287 | return p.temp.items[p.idx..]; | | |
| 288 | } | | |
| 289 | | | |
| 290 | pub fn eat(p: *Parser, comptime test_s: string) !?void { | | |
| 291 | if (test_s.len == 1) { | | |
| 292 | _ = try p.eatByte(test_s[0]); | | |
| 293 | return; | | |
| 294 | } | | |
| 295 | try p.peekAmt(test_s.len) orelse return null; | | |
| 296 | if (std.mem.eql(u8, p.slice()[0..test_s.len], test_s)) { | | |
| 297 | p.idx += test_s.len; | | |
| 298 | return; | | |
| 299 | } | | |
| 300 | return null; | | |
| 301 | } | | |
| 302 | | | |
| 303 | fn peekAmt(p: *Parser, amt: usize) !?void { | | |
| 304 | if (p.avail() >= amt) return; | | |
| 305 | const buf_size = std.mem.page_size; | | |
| 306 | const diff_amt = amt - p.avail(); | | |
| 307 | std.debug.assert(diff_amt <= buf_size); | | |
| 308 | var buf: [buf_size]u8 = undefined; | | |
| 309 | const len = try p.any.readAll(&buf); | | |
| 310 | if (len == 0) p.end = true; | | |
| 311 | if (len == 0) return null; | | |
| 312 | try p.temp.appendSlice(p.arena, buf[0..len]); | | |
| 313 | if (amt > len) return null; | | |
| 314 | } | | |
| 315 | | | |
| 316 | pub fn eatByte(p: *Parser, test_c: u8) !?u8 { | | |
| 317 | const t = tracer.trace(@src(), " '{c}'", .{test_c}); | | |
| 318 | defer t.end(); | | |
| 319 | | | |
| 320 | try p.peekAmt(1) orelse return null; | | |
| 321 | if (p.slice()[0] == test_c) { | | |
| 322 | p.idx += 1; | | |
| 323 | return test_c; | | |
| 324 | } | | |
| 325 | return null; | | |
| 326 | } | | |
| 327 | | | |
| 328 | pub fn eatRange(p: *Parser, comptime from: u8, comptime to: u8) !?u8 { | | |
| 329 | const t = tracer.trace(@src(), " ({d},{d})", .{ from, to }); | | |
| 330 | defer t.end(); | | |
| 331 | | | |
| 332 | try p.peekAmt(1) orelse return null; | | |
| 333 | if (p.slice()[0] >= from and p.slice()[0] <= to) { | | |
| 334 | defer p.idx += 1; | | |
| 335 | return p.slice()[0]; | | |
| 336 | } | | |
| 337 | return null; | | |
| 338 | } | | |
| 339 | | | |
| 340 | pub fn eatAnyScalar(p: *Parser, test_s: string) !?u8 { | | |
| 341 | const t = tracer.trace(@src(), " ({s})", .{test_s}); | | |
| 342 | defer t.end(); | | |
| 343 | | | |
| 344 | std.debug.assert(extras.matchesAll(u8, test_s, std.ascii.isASCII)); | | |
| 345 | try p.peekAmt(1) orelse return null; | | |
| 346 | if (std.mem.indexOfScalar(u8, test_s, p.slice()[0])) |idx| { | | |
| 347 | p.idx += 1; | | |
| 348 | return test_s[idx]; | | |
| 349 | } | | |
| 350 | return null; | | |
| 351 | } | | |
| 352 | | | |
| 353 | pub fn shift(p: *Parser) !u21 { | | |
| 354 | try p.peekAmt(1) orelse return error.EndOfStream; | | |
| 355 | const len = std.unicode.utf8ByteSequenceLength(p.slice()[0]) catch return error.MalformedJson; | | |
| 356 | try p.peekAmt(len) orelse return error.EndOfStream; | | |
| 357 | defer p.idx += len; | | |
| 358 | return std.unicode.utf8Decode(p.slice()[0..len]) catch return error.MalformedJson; | | |
| 359 | } | | |
| 360 | | | |
| 361 | pub fn shiftBytesN(p: *Parser, comptime n: usize) ![n]u8 { | | |
| 362 | try p.peekAmt(n) orelse return error.EndOfStream; | | |
| 363 | defer p.idx += n; | | |
| 364 | return p.slice()[0..n].*; | | |
| 365 | } | | |
| 366 | | 273 | |
| 367 | // tag(u8) + len(u32) + member_keys(N * u32) + member_values(N * u32) | 274 | // tag(u8) + len(u32) + member_keys(N * u32) + member_values(N * u32) |
| 368 | pub fn addObject(p: *Parser, alloc: std.mem.Allocator, members: *const ObjectHashMap) !ValueIndex { | 275 | pub fn addObject(p: *Parser, alloc: std.mem.Allocator, members: *const ObjectHashMap) !ValueIndex { |
| 369 | const t = tracer.trace(@src(), "({d})", .{members.entries.len}); | 276 | const t = tracer.trace(@src(), "({d})", .{members.entries.len}); |
| 370 | defer t.end(); | 277 | defer t.end(); |
| 371 | | 278 | |
| 372 | const r = p.extras.items.len; | 279 | const r = p.parser.data.items.len; |
| 373 | const l = members.entries.len; | 280 | const l = members.entries.len; |
| 374 | if (l > std.math.maxInt(u32)) return error.MalformedJson; | 281 | if (l > std.math.maxInt(u32)) return error.MalformedJson; |
| 375 | try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4 * 2)); | 282 | try p.parser.data.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4 * 2)); |
| 376 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.object)); | 283 | p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.object)); |
| 377 | p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l)))); | 284 | p.parser.data.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l)))); |
| 378 | p.extras.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.keys())); | 285 | p.parser.data.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.keys())); |
| 379 | p.extras.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.values())); | 286 | p.parser.data.appendSliceAssumeCapacity(std.mem.sliceAsBytes(members.values())); |
| 380 | return @enumFromInt(r); | 287 | return @enumFromInt(r); |
| 381 | } | 288 | } |
| 382 | | 289 | |
| ... | @@ -385,33 +292,19 @@ const Parser = struct { | ... | @@ -385,33 +292,19 @@ const Parser = struct { |
| 385 | const t = tracer.trace(@src(), "({d})", .{items.len}); | 292 | const t = tracer.trace(@src(), "({d})", .{items.len}); |
| 386 | defer t.end(); | 293 | defer t.end(); |
| 387 | | 294 | |
| 388 | const r = p.extras.items.len; | 295 | const r = p.parser.data.items.len; |
| 389 | const l = items.len; | 296 | const l = items.len; |
| 390 | if (l > std.math.maxInt(u32)) return error.MalformedJson; | 297 | if (l > std.math.maxInt(u32)) return error.MalformedJson; |
| 391 | try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4)); | 298 | try p.parser.data.ensureUnusedCapacity(alloc, 1 + 4 + (l * 4)); |
| 392 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.array)); | 299 | p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.array)); |
| 393 | p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l)))); | 300 | p.parser.data.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l)))); |
| 394 | p.extras.appendSliceAssumeCapacity(std.mem.sliceAsBytes(items)); | 301 | p.parser.data.appendSliceAssumeCapacity(std.mem.sliceAsBytes(items)); |
| 395 | return @enumFromInt(r); | 302 | return @enumFromInt(r); |
| 396 | } | 303 | } |
| 397 | | 304 | |
| 398 | // tag(u8) + len(u32) + bytes(N) | 305 | // tag(u8) + len(u32) + bytes(N) |
| 399 | pub fn addStr(p: *Parser, alloc: std.mem.Allocator, str: string) !StringIndex { | 306 | pub fn addStr(p: *Parser, alloc: std.mem.Allocator, str: string) !StringIndex { |
| 400 | const t = tracer.trace(@src(), "({d})", .{str.len}); | 307 | return @enumFromInt(try p.parser.addStr(alloc, str)); |
| 401 | defer t.end(); | | |
| 402 | | | |
| 403 | const adapter: Adapter = .{ .p = p }; | | |
| 404 | const res = try p.strings_map.getOrPutAdapted(alloc, str, adapter); | | |
| 405 | if (res.found_existing) return res.value_ptr.*; | | |
| 406 | errdefer p.strings_map.orderedRemoveAt(res.index); | | |
| 407 | const r = p.extras.items.len; | | |
| 408 | const l = str.len; | | |
| 409 | try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + l); | | |
| 410 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.string)); | | |
| 411 | p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l)))); | | |
| 412 | p.extras.appendSliceAssumeCapacity(str); | | |
| 413 | res.value_ptr.* = @enumFromInt(r); | | |
| 414 | return @enumFromInt(r); | | |
| 415 | } | 308 | } |
| 416 | | 309 | |
| 417 | const Adapter = struct { | 310 | const Adapter = struct { |
| ... | @@ -439,12 +332,12 @@ const Parser = struct { | ... | @@ -439,12 +332,12 @@ const Parser = struct { |
| 439 | const res = try p.numbers_map.getOrPutAdapted(alloc, v, adapter); | 332 | const res = try p.numbers_map.getOrPutAdapted(alloc, v, adapter); |
| 440 | if (res.found_existing) return @enumFromInt(@intFromEnum(res.value_ptr.*)); | 333 | if (res.found_existing) return @enumFromInt(@intFromEnum(res.value_ptr.*)); |
| 441 | errdefer p.numbers_map.orderedRemoveAt(res.index); | 334 | errdefer p.numbers_map.orderedRemoveAt(res.index); |
| 442 | const r = p.extras.items.len; | 335 | const r = p.parser.data.items.len; |
| 443 | const l = v.len; | 336 | const l = v.len; |
| 444 | try p.extras.ensureUnusedCapacity(alloc, 1 + 4 + l); | 337 | try p.parser.data.ensureUnusedCapacity(alloc, 1 + 4 + l); |
| 445 | p.extras.appendAssumeCapacity(@intFromEnum(Value.Tag.number)); | 338 | p.parser.data.appendAssumeCapacity(@intFromEnum(Value.Tag.number)); |
| 446 | p.extras.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l)))); | 339 | p.parser.data.appendSliceAssumeCapacity(&std.mem.toBytes(@as(u32, @intCast(l)))); |
| 447 | p.extras.appendSliceAssumeCapacity(v); | 340 | p.parser.data.appendSliceAssumeCapacity(v); |
| 448 | res.value_ptr.* = @enumFromInt(r); | 341 | res.value_ptr.* = @enumFromInt(r); |
| 449 | return @enumFromInt(r); | 342 | return @enumFromInt(r); |
| 450 | } | 343 | } |
| ... | @@ -462,19 +355,12 @@ const Parser = struct { | ... | @@ -462,19 +355,12 @@ const Parser = struct { |
| 462 | pub fn eql(ctx: @This(), a: string, _: string, b_index: usize) bool { | 355 | pub fn eql(ctx: @This(), a: string, _: string, b_index: usize) bool { |
| 463 | const sidx = ctx.p.numbers_map.values()[b_index]; | 356 | const sidx = ctx.p.numbers_map.values()[b_index]; |
| 464 | const i: u32 = @intFromEnum(sidx); | 357 | const i: u32 = @intFromEnum(sidx); |
| 465 | std.debug.assert(@as(Value.Tag, @enumFromInt(ctx.p.extras.items[i])) == .number); | 358 | std.debug.assert(@as(Value.Tag, @enumFromInt(ctx.p.parser.data.items[i])) == .number); |
| 466 | const l: u32 = @bitCast(ctx.p.extras.items[i..][1..][0..4].*); | 359 | const l: u32 = @bitCast(ctx.p.parser.data.items[i..][1..][0..4].*); |
| 467 | const b = ctx.p.extras.items[i..][1..][4..][0..l]; | 360 | const b = ctx.p.parser.data.items[i..][1..][4..][0..l]; |
| 468 | return std.mem.eql(u8, a, b); | 361 | return std.mem.eql(u8, a, b); |
| 469 | } | 362 | } |
| 470 | }; | 363 | }; |
| 471 | | | |
| 472 | pub fn getStr(p: *const Parser, sidx: StringIndex) string { | | |
| 473 | const i: u32 = @intFromEnum(sidx); | | |
| 474 | std.debug.assert(@as(Value.Tag, @enumFromInt(p.extras.items[i])) == .string); | | |
| 475 | const l: u32 = @bitCast(p.extras.items[i..][1..][0..4].*); | | |
| 476 | return p.extras.items[i..][1..][4..][0..l]; | | |
| 477 | } | | |
| 478 | }; | 364 | }; |
| 479 | | 365 | |
| 480 | pub threadlocal var doc: ?*const Document = null; | 366 | pub threadlocal var doc: ?*const Document = null; |