authorgravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2021-08-13 17:52:29 -07:00
committergravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2021-08-13 17:52:29 -07:00
log8e154277a37b31d312596f6108f123eb597846da
tree291559c76af1f87fe8ce59d8ff8a412a0f108b02
parentbcb022592b5751ff9acd5b9ac18d0dfea8fe447f

add line and pos to token struct

2 files changed, 34 insertions(+), 11 deletions(-)

src/astgen.zig+9-4
...@@ -84,7 +84,7 @@ const Parser = struct {...@@ -84,7 +84,7 @@ const Parser = struct {
84 }84 }
8585
86 fn tryEatSymbol(comptime self: *Parser, comptime needle: string) bool {86 fn tryEatSymbol(comptime self: *Parser, comptime needle: string) bool {
87 switch (self.tokens[self.index]) {87 switch (self.tokens[self.index].data) {
88 .symbol => |sym| {88 .symbol => |sym| {
89 if (std.mem.eql(u8, sym, needle)) {89 if (std.mem.eql(u8, sym, needle)) {
90 self.index += 1;90 self.index += 1;
...@@ -126,7 +126,7 @@ const Parser = struct {...@@ -126,7 +126,7 @@ const Parser = struct {
126 }126 }
127127
128 pub fn doValue(comptime self: *Parser) Value {128 pub fn doValue(comptime self: *Parser) Value {
129 if (self.tokens[self.index] == .string) {129 if (self.tokens[self.index].data == .string) {
130 return Value{ .string = self.eat(.string) };130 return Value{ .string = self.eat(.string) };
131 }131 }
132 if (self.tryEatSymbol("{")) {132 if (self.tryEatSymbol("{")) {
...@@ -193,8 +193,13 @@ const Parser = struct {...@@ -193,8 +193,13 @@ const Parser = struct {
193 return ret;193 return ret;
194 }194 }
195195
196 fn eat(comptime self: *Parser, comptime typ: std.meta.Tag(Token)) string {196 fn eat(comptime self: *Parser, comptime typ: std.meta.Tag(Token.Data)) string {
197 defer self.index += 1;197 defer self.index += 1;
198 return @field(self.tokens[self.index], @tagName(typ));198 const tok = self.tokens[self.index];
199 const tag = std.meta.activeTag(tok.data);
200 if (tag != typ) {
201 @compileError(std.fmt.comptimePrint("pek: file:{d}:{d}: expected {s}, found {s}", .{ tok.line, tok.pos, @tagName(tag), @tagName(typ) }));
202 }
203 return @field(tok.data, @tagName(typ));
199 }204 }
200};205};
src/tokenize.zig+25-7
...@@ -4,12 +4,18 @@ const string = []const u8;...@@ -4,12 +4,18 @@ const string = []const u8;
4//4//
5//5//
66
7pub const Token = union(enum) {7pub const Token = struct {
8 word: string,8 data: Data,
9 symbol: string,9 line: usize,
10 string: string,10 pos: usize,
1111
12 pub const skippedChars = &[_]u8{ ' ', '\n', '\t', '\r' };12 pub const skippedChars = &[_]u8{ ' ', '\n', '\t', '\r' };
13
14 pub const Data = union(enum) {
15 word: string,
16 symbol: string,
17 string: string,
18 };
13};19};
1420
15//21//
...@@ -58,7 +64,11 @@ pub fn do(comptime input: string, comptime symbols: []const u8) []const Token {...@@ -58,7 +64,11 @@ pub fn do(comptime input: string, comptime symbols: []const u8) []const Token {
58 }64 }
59 if (mode == 2) {65 if (mode == 2) {
60 if (c == input[start]) {66 if (c == input[start]) {
61 ret = ret ++ &[_]Token{.{ .string = input[start .. i + 1] }};67 ret = ret ++ &[_]Token{.{
68 .data = .{ .string = input[start .. i + 1] },
69 .line = line,
70 .pos = pos,
71 }};
62 start = i + 1;72 start = i + 1;
63 end = i;73 end = i;
64 mode = 0;74 mode = 0;
...@@ -84,7 +94,11 @@ pub fn do(comptime input: string, comptime symbols: []const u8) []const Token {...@@ -84,7 +94,11 @@ pub fn do(comptime input: string, comptime symbols: []const u8) []const Token {
84 if (shouldFlush) {94 if (shouldFlush) {
85 if (mode == 0) {95 if (mode == 0) {
86 if (end - start > 0) {96 if (end - start > 0) {
87 ret = ret ++ &[_]Token{.{ .word = input[start..end] }};97 ret = ret ++ &[_]Token{.{
98 .data = .{ .word = input[start..end] },
99 .line = line,
100 .pos = pos,
101 }};
88 start = i;102 start = i;
89 end = i;103 end = i;
90 }104 }
...@@ -93,7 +107,11 @@ pub fn do(comptime input: string, comptime symbols: []const u8) []const Token {...@@ -93,7 +107,11 @@ pub fn do(comptime input: string, comptime symbols: []const u8) []const Token {
93 end += 1;107 end += 1;
94 }108 }
95 if (std.mem.indexOfScalar(u8, symbols, c)) |_| {109 if (std.mem.indexOfScalar(u8, symbols, c)) |_| {
96 ret = ret ++ &[_]Token{.{ .symbol = s }};110 ret = ret ++ &[_]Token{.{
111 .data = .{ .symbol = s },
112 .line = line,
113 .pos = pos,
114 }};
97 start += 1;115 start += 1;
98 end += 1;116 end += 1;
99 }117 }