1import { closeSync, openSync } from "node:fs";
2
3const unicode_version = "17.0.0";
4const encoder = new TextEncoder();
5
6function stringEscape(s?: string) {
7 if (!s) return "";
8 return Array.from(encoder.encode(s.replace(/\\u([0-9A-F]{4})/g, (s) => String.fromCodePoint(parseInt(s.slice(2), 16)))))
9 .map((x) => {
10 if (x === 0x09) return "\\t";
11 if (x === 0x0a) return "\\n";
12 if (x === 0x0d) return "\\r";
13 if (x === 0x20) return " ";
14 if (x === 0x21) return "!";
15 if (x === 0x22) return `\\"`;
16 if (x >= 0x23 && x <= 0x26) return String.fromCodePoint(x);
17 if (x === 0x27) return "'";
18 if (x >= 0x28 && x <= 0x5b) return String.fromCodePoint(x);
19 if (x === 0x5c) return "\\\\";
20 if (x >= 0x5d && x <= 0x7e) return String.fromCodePoint(x);
21 return `\\x${x.toString(16).padStart(2, "0")}`;
22 })
23 .join("");
24}
25const E = stringEscape;
26
27{
28 const source = `https://www.unicode.org/Public/${unicode_version}/idna/IdnaMappingTable.txt`;
29 const response = await fetch(source);
30 const data = await response.text();
31 const lines = data.split("\n").filter((x) => !x.startsWith("#") && x.length > 0);
32 const lines_clear = lines.map((x) => x.split("#")[0]!);
33 const cols = lines_clear.map((x) => x.split(";").map((y) => y.trim()));
34
35 closeSync(openSync("./table.zig", "w", 0o777));
36 const f = Bun.file("./table.zig");
37 const w = f.writer();
38
39 w.write(`// This file is part of Unicode IDNA Compatibility Processing\n`);
40 w.write(`// For documentation, see http://www.unicode.org/reports/tr46/\n`);
41 w.write(`//\n`);
42 w.write(`\n`);
43 w.write(`// Based on the source file: ${source}\n`);
44 w.write(`//\n`);
45 w.write(`\n`);
46 w.write(`pub const Row = struct {\n`);
47 w.write(` cp: u21,\n`);
48 w.write(` status: Status,\n`);
49 w.write(` mapping: []const u21,\n`);
50 w.write(` status2: Status2008,\n`);
51 w.write(`};\n`);
52 w.write(`\n`);
53 w.write(`pub const RowRange = struct {\n`);
54 w.write(` from: u21,\n`);
55 w.write(` to: u21,\n`);
56 w.write(` status: Status,\n`);
57 w.write(` mapping: []const u21,\n`);
58 w.write(` status2: Status2008,\n`);
59 w.write(`};\n`);
60 w.write(`\n`);
61 w.write(`pub const Status = enum {\n`);
62 w.write(` valid,\n`);
63 w.write(` ignored,\n`);
64 w.write(` mapped,\n`);
65 w.write(` deviation,\n`);
66 w.write(` disallowed,\n`);
67 w.write(`};\n`);
68 w.write(`\n`);
69 w.write(`pub const Status2008 = enum {\n`);
70 w.write(` none,\n`);
71 w.write(` NV8,\n`);
72 w.write(` XV8,\n`);
73 w.write(`};\n`);
74 w.write(`\n`);
75 w.write(`pub const data = [_]Row{\n`);
76 {
77 for (const c of cols.filter((x) => !x[0]!.includes(".."))) {
78 w.write(` .{ .cp = 0x${c[0]},`);
79 w.write(` .status = .${c[1]},`);
80 // prettier-ignore
81 w.write(` .mapping = &.{${c[2]?.split(" ").filter(x => x.length > 0).map((x) => `0x${x}`).join(",") ?? ""}},`,);
82 w.write(` .status2 = .${c[3] ?? "none"},`);
83 w.write(` },\n`);
84 }
85 }
86 w.write(`};\n`);
87 w.write(`\n`);
88 w.write(`pub const data_range = [_]RowRange{\n`);
89 {
90 for (const c of cols.filter((x) => x[0]!.includes(".."))) {
91 w.write(` .{ .from = 0x${c[0]!.split("..")[0]}, .to = 0x${c[0]!.split("..")[1]},`);
92 w.write(` .status = .${c[1]},`);
93 // prettier-ignore
94 w.write(` .mapping = &.{${c[2]?.split(" ").filter(x => x.length > 0).map((x) => `0x${x}`).join(",") ?? ""}},`,);
95 w.write(` .status2 = .${c[3] ?? "none"},`);
96 w.write(` },\n`);
97 }
98 }
99 w.write(`};\n`);
100 w.flush();
101}
102
103{
104 const source = `https://www.unicode.org/Public/${unicode_version}/idna/Idna2008.txt`;
105 const response = await fetch(source);
106 const data = await response.text();
107 const lines = data.split("\n").filter((x) => !x.startsWith("#") && x.length > 0);
108 const lines_clear = lines.map((x) => x.split("#")[0]!);
109 const cols = lines_clear.map((x) => x.split(";").map((y) => y.trim()));
110
111 closeSync(openSync("./2008.zig", "w", 0o777));
112 const f = Bun.file("./2008.zig");
113 const w = f.writer();
114
115 w.write(`// This file is part of Unicode IDNA Compatibility Processing\n`);
116 w.write(`// For documentation, see http://www.unicode.org/reports/tr46/\n`);
117 w.write(`//\n`);
118 w.write(`\n`);
119 w.write(`// Based on the source file: ${source}\n`);
120 w.write(`//\n`);
121 w.write(`\n`);
122 w.write(`pub const Prop = struct {\n`);
123 w.write(` cp: u21,\n`);
124 w.write(` category: Category,\n`);
125 w.write(`};\n`);
126 w.write(`\n`);
127 w.write(`pub const PropRange = struct {\n`);
128 w.write(` from: u21,\n`);
129 w.write(` to: u21,\n`);
130 w.write(` category: Category,\n`);
131 w.write(`};\n`);
132 w.write(`\n`);
133 w.write(`pub const Category = enum {\n`);
134 w.write(` pvalid,\n`);
135 w.write(` contextj,\n`);
136 w.write(` contexto,\n`);
137 w.write(` disallowed,\n`);
138 w.write(` unassigned,\n`);
139 w.write(`};\n`);
140 w.write(`\n`);
141 w.write(`pub const data = [_]Prop{\n`);
142 {
143 for (const c of cols.filter((x) => !x[0]!.includes(".."))) {
144 w.write(` .{ .cp = 0x${c[0]}, .category = .${c[1]!.toLowerCase()} },\n`);
145 }
146 }
147 w.write(`};\n`);
148 w.write(`\n`);
149 w.write(`pub const data_range = [_]PropRange{\n`);
150 {
151 for (const c of cols.filter((x) => x[0]!.includes(".."))) {
152 w.write(` .{ .from = 0x${c[0]!.split("..")[0]}, .to = 0x${c[0]!.split("..")[1]}, .category = .${c[1]!.toLowerCase()} },\n`);
153 }
154 }
155 w.write(`};\n`);
156 w.flush();
157}
158
159{
160 const source = `https://www.unicode.org/Public/${unicode_version}/idna/IdnaTestV2.txt`;
161 const response = await fetch(source);
162 const data = await response.text();
163 const lines = data.split("\n").filter((x) => !x.startsWith("#") && x.length > 0);
164 const lines_clear = lines.map((x) => x.split("#")[0]!);
165 const cols = lines_clear.map((x) => x.split(";").map((y) => y.trim()));
166
167 closeSync(openSync("./testv2.zig", "w", 0o777));
168 const f = Bun.file("./testv2.zig");
169 const w = f.writer();
170
171 w.write(`// This file is part of Unicode IDNA Compatibility Processing\n`);
172 w.write(`// For documentation, see http://www.unicode.org/reports/tr46/\n`);
173 w.write(`//\n`);
174 w.write(`\n`);
175 w.write(`// Based on the source file: ${source}\n`);
176 w.write(`//\n`);
177 w.write(`// zig fmt: off\n`);
178 w.write(`\n`);
179 w.write(`const std = @import("std");\n`);
180 w.write(`const idna = @import("unicode-idna");\n`);
181 w.write(`const expect = @import("expect").expect;\n`);
182
183 w.write(`
184fn toUnicodePass(
185 source: []const u8,
186 expected: []const u8,
187) !void {
188 const allocator = std.testing.allocator;
189 const result = try idna.ToUnicode(allocator, source, true, true, true, true, false, false);
190 defer allocator.free(result);
191 try expect(result).toEqualString(expected);
192}
193
194fn toUnicodeFail(
195 source: []const u8,
196) !void {
197 const allocator = std.testing.allocator;
198 const result = idna.ToUnicode(allocator, source, true, true, true, true, false, false) catch |err| switch (err) {
199 error.IDNAFailure => return,
200 error.OutOfMemory => return error.OutOfMemory,
201 };
202 defer allocator.free(result);
203 return error.ShouldHaveFailed;
204}
205
206fn toAsciiPass(
207 source: []const u8,
208 expected: []const u8,
209 Transitional_Processing: bool,
210) !void {
211 const allocator = std.testing.allocator;
212 const result = try idna.ToASCII(allocator, source, true, true, true, true, Transitional_Processing, true, false);
213 defer allocator.free(result);
214 try expect(result).toEqualString(expected);
215}
216
217fn toAsciiFail(
218 source: []const u8,
219 Transitional_Processing: bool,
220) !void {
221 const allocator = std.testing.allocator;
222 const result = idna.ToASCII(allocator, source, true, true, true, true, Transitional_Processing, true, false) catch |err| switch (err) {
223 error.IDNAFailure => return,
224 error.OutOfMemory => return error.OutOfMemory,
225 };
226 defer allocator.free(result);
227 return error.ShouldHaveFailed;
228}
229`);
230 w.write(`\n`);
231
232 for (const i of cols) {
233 const source = i[0];
234 const toUnicode = i[1] || source;
235 const toUnicodeStatus = i[2] || "[]";
236 const toAsciiN = i[3] || toUnicode;
237 const toAsciiNStatus = i[4] || toUnicodeStatus;
238 const toAsciiT = i[5] || toAsciiN;
239 const toAsciiTStatus = i[6] || toAsciiNStatus;
240
241 if (toUnicodeStatus === "[]") w.write(`test { try toUnicodePass("${E(source)}", "${E(toUnicode)}"); }\n`);
242 if (toAsciiNStatus === "[]") w.write(`test { try toAsciiPass("${E(source)}", "${E(toAsciiN)}", false); }\n`);
243 if (toAsciiTStatus === "[]") w.write(`test { try toAsciiPass("${E(source)}", "${E(toAsciiT)}", true); }\n`);
244
245 if (toUnicodeStatus !== "[]" && toUnicodeStatus !== "[X4_2]") w.write(`test { try toUnicodeFail("${E(source)}"); } // ${toUnicodeStatus}\n`);
246 if (toAsciiNStatus !== "[]") w.write(`test { try toAsciiFail("${E(source)}", false); } // ${toAsciiNStatus}\n`);
247 if (toAsciiTStatus !== "[]") w.write(`test { try toAsciiFail("${E(source)}", true); } // ${toAsciiTStatus}\n`);
248 }
249
250 w.flush();
251}