1const std = @import("std");
2const string = []const u8;
3
4//
5//
6
7pub const Token = struct {
8 data: Data,
9 line: usize,
10 pos: usize,
11
12 pub const skippedChars = &[_]u8{ ' ', '\n', '\t', '\r' };
13
14 pub const Data = union(enum) {
15 word: string,
16 symbol: string,
17 string: string,
18 };
19};
20
21//
22//
23
24pub fn do(comptime input: string, comptime symbols: []const u8) []const Token {
25 var ret: []const Token = &[_]Token{};
26
27 var line = 1;
28 var pos = 1;
29
30 var start = 0;
31 var end = 0;
32 var mode = 0;
33
34 @setEvalBranchQuota(1000000);
35 if (!@inComptime()) @compileError("must be at comptime!");
36
37 inline for (input, 0..) |c, i| {
38 const s = &[_]u8{c};
39
40 var shouldFlush: bool = undefined;
41
42 blk: {
43 if (mode == 0) {
44 if (c == '/' and input[i + 1] == '/') {
45 mode = 1;
46 shouldFlush = false;
47 break :blk;
48 }
49 if (c == '"') {
50 mode = 2;
51 shouldFlush = false;
52 break :blk;
53 }
54 if (c == '\'') {
55 mode = 2;
56 shouldFlush = false;
57 break :blk;
58 }
59 }
60 if (mode == 1) {
61 if (c == '\n') {
62 // skip comments
63 // f(v.handle(TTCom, in[s:i]))
64 start = i;
65 end = i;
66 mode = 0;
67 }
68 shouldFlush = c == '\n';
69 break :blk;
70 }
71 if (mode == 2) {
72 if (c == input[start]) {
73 const data = input[start .. i + 1][0..].*;
74 ret = ret ++ &[_]Token{.{
75 .data = .{ .string = &data },
76 .line = line,
77 .pos = pos,
78 }};
79 start = i + 1;
80 end = i;
81 mode = 0;
82 }
83 shouldFlush = false;
84 break :blk;
85 }
86 if (std.mem.indexOfScalar(u8, Token.skippedChars, c)) |_| {
87 shouldFlush = true;
88 break :blk;
89 }
90 if (std.mem.indexOfScalar(u8, symbols, c)) |_| {
91 shouldFlush = true;
92 break :blk;
93 }
94 shouldFlush = false;
95 break :blk;
96 }
97
98 if (!shouldFlush) {
99 end += 1;
100 }
101 if (shouldFlush) {
102 if (mode == 0) {
103 if (end - start > 0) {
104 const data = input[start..end][0..].*;
105 ret = ret ++ &[_]Token{.{
106 .data = .{ .word = &data },
107 .line = line,
108 .pos = pos,
109 }};
110 start = i;
111 end = i;
112 }
113 if (std.mem.indexOfScalar(u8, Token.skippedChars, c)) |_| {
114 start += 1;
115 end += 1;
116 }
117 if (std.mem.indexOfScalar(u8, symbols, c)) |_| {
118 ret = ret ++ &[_]Token{.{
119 .data = .{ .symbol = s },
120 .line = line,
121 .pos = pos,
122 }};
123 start += 1;
124 end += 1;
125 }
126 }
127 }
128
129 pos += 1;
130 if (c != '\n') continue;
131 line += 1;
132 pos = 1;
133 }
134
135 return ret;
136}