authorgravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2026-08-26 17:24:03-07:00
committergravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2026-08-26 17:24:03-07:00
log7b11277ab73bad88d8ec336ffae899f5c72ce07b
tree03687add7c0840e72d1a7eb88aa8e26de318f450
parent0f0a8e02e13be7d19ec182944070420e1598d8e0
signature Signed by SSH key SHA256:4hHJbtBRU58AYXwjL7fkz2fnQHdiye8x1QpTCQ0sHNw

don't use udc.unicode_data.data directly

doing so caused massive amounts of bloat (>6mb)

2 files changed, 67 insertions(+), 61 deletions(-)

idna.zig+40-35
......@@ -157,7 +157,7 @@ fn Processing(
157157 while (it.nextCodepointSlice()) |s| {
158158 const c = std.unicode.utf8Decode(s) catch unreachable;
159159 const p = ucd.unicode_data.find(c);
160 switch (p[4]) {
160 switch (ucd.unicode_data.data_bc[p]) {
161161 .R, .AL, .AN => domain_is_bidi = true,
162162 else => {},
163163 }
......@@ -229,11 +229,13 @@ fn Validity_Criteria(
229229 const first_l = std.unicode.utf8ByteSequenceLength(first_b) catch unreachable;
230230 const first_c = std.unicode.utf8Decode(label[0..first_l]) catch unreachable;
231231 const first_p = ucd.unicode_data.find(first_c);
232 if (first_p[2] == .M) return error.IDNAFailure;
233 if (first_p[2] == .Combining_Mark) return error.IDNAFailure;
234 if (first_p[2] == .Mc) return error.IDNAFailure;
235 if (first_p[2] == .Me) return error.IDNAFailure;
236 if (first_p[2] == .Mn) return error.IDNAFailure;
232 const first_gc = ucd.unicode_data.data_gc[first_p];
233 const first_bc = ucd.unicode_data.data_bc[first_p];
234 if (first_gc == .M) return error.IDNAFailure;
235 if (first_gc == .Combining_Mark) return error.IDNAFailure;
236 if (first_gc == .Mc) return error.IDNAFailure;
237 if (first_gc == .Me) return error.IDNAFailure;
238 if (first_gc == .Mn) return error.IDNAFailure;
237239
238240 // 7. Each code point in the label must only have certain Status values according to Section 5, IDNA Mapping Table:
239241 var it = std.unicode.Utf8View.initUnchecked(label).iterator();
......@@ -241,8 +243,8 @@ fn Validity_Criteria(
241243 var prev_cp: u21 = 0;
242244
243245 // 8. If CheckJoiners, the label must satisify the ContextJ rules from RFC 5892 Appendix A.
244 var point_2before: ucd.unicode_data.Codepoint = ucd.unicode_data.data[0];
245 var point_before: ucd.unicode_data.Codepoint = ucd.unicode_data.data[0];
246 var point_2before: usize = 0;
247 var point_before: usize = 0;
246248 var zwnj_state: enum { start, dual_joining, transparent1, zwnj, transparent2, right_joining } = .start;
247249 var check_zwnj_state: ?bool = null;
248250
......@@ -257,7 +259,7 @@ fn Validity_Criteria(
257259 var seen_an = false;
258260 var last_bidi_class: ucd.BidiClass = .NSM;
259261 if (CheckBidi) {
260 switch (first_p[4]) {
262 switch (first_bc) {
261263 .L => {},
262264 .R, .AL => {},
263265 else => return error.IDNAFailure,
......@@ -274,23 +276,25 @@ fn Validity_Criteria(
274276 for (&ucd.derived_normalization_props.nfc_qc_n.data) |d| if (cp == d) return error.IDNAFailure;
275277 for (&ucd.derived_normalization_props.nfc_qc_n.data_range) |d| if (cp >= d.from and cp <= d.to) return error.IDNAFailure;
276278 for (&ucd.derived_normalization_props.nfc_qc_m.data) |d| if (cp == d) {
277 for (&ucd.unicode_data.data) |r| {
278 if (r[5] != .__canonical) continue;
279 if (r[6].len != 2) continue;
280 if (r[6][0] != prev_cp) continue;
281 if (r[6][1] != cp) continue;
282 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, r[0], extras.compareFnBasic(u21)) != null) continue;
279 const ud = ucd.unicode_data;
280 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map) |_cp, dc, dm| {
281 if (dc != .__canonical) continue;
282 if (dm.len != 2) continue;
283 if (dm[0] != prev_cp) continue;
284 if (dm[1] != cp) continue;
285 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, _cp, extras.compareFnBasic(u21)) != null) continue;
283286 // if (r[3] != 0) continue;
284287 return error.IDNAFailure;
285288 }
286289 };
287290 for (&ucd.derived_normalization_props.nfc_qc_m.data_range) |d| if (cp >= d.from and cp <= d.to) {
288 for (&ucd.unicode_data.data) |r| {
289 if (r[5] != .__canonical) continue;
290 if (r[6].len != 2) continue;
291 if (r[6][0] != prev_cp) continue;
292 if (r[6][1] != cp) continue;
293 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, r[0], extras.compareFnBasic(u21)) != null) continue;
291 const ud = ucd.unicode_data;
292 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map) |cpp, dc, dm| {
293 if (dc != .__canonical) continue;
294 if (dm.len != 2) continue;
295 if (dm[0] != prev_cp) continue;
296 if (dm[1] != cp) continue;
297 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cpp, extras.compareFnBasic(u21)) != null) continue;
294298 // if (r[3] != 0) continue;
295299 return error.IDNAFailure;
296300 }
......@@ -316,12 +320,13 @@ fn Validity_Criteria(
316320 }
317321
318322 const p = ucd.unicode_data.find(cp);
323 const p_bc = ucd.unicode_data.data_bc[p];
319324
320325 if (CheckJoiners) {
321326 // Appendix A.1. ZERO WIDTH NON-JOINER
322327 if (cp == 0x200C) {
323328 if (check_zwnj_state == null) check_zwnj_state = true;
324 if (point_before[3] == 9) check_zwnj_state = false;
329 if (ucd.unicode_data.data_ccc[point_before] == 9) check_zwnj_state = false;
325330 }
326331 switch (zwnj_state) {
327332 .start => {
......@@ -377,15 +382,15 @@ fn Validity_Criteria(
377382 },
378383 }
379384 // Appendix A.2. ZERO WIDTH JOINER
380 if (cp == 0x200D and point_before[3] != 9) return error.IDNAFailure;
385 if (cp == 0x200D and ucd.unicode_data.data_ccc[point_before] != 9) return error.IDNAFailure;
381386 // Appendix A.3. MIDDLE DOT
382 if (point_before[0] == 0x00B7 and !(point_2before[0] == 0x006C and cp == 0x006C)) return error.IDNAFailure;
387 if (ucd.unicode_data.data_code[point_before] == 0x00B7 and !(ucd.unicode_data.data_code[point_2before] == 0x006C and cp == 0x006C)) return error.IDNAFailure;
383388 // Appendix A.4. GREEK LOWER NUMERAL SIGN (KERAIA)
384 if (point_before[0] == 0x0375 and !(Script(cp) == .Greek)) return error.IDNAFailure;
389 if (ucd.unicode_data.data_code[point_before] == 0x0375 and !(Script(cp) == .Greek)) return error.IDNAFailure;
385390 // Appendix A.5. HEBREW PUNCTUATION GERESH
386 if (cp == 0x05F3 and !(Script(point_before[0]) == .Hebrew)) return error.IDNAFailure;
391 if (cp == 0x05F3 and !(Script(ucd.unicode_data.data_code[point_before]) == .Hebrew)) return error.IDNAFailure;
387392 // Appendix A.6. HEBREW PUNCTUATION GERSHAYIM
388 if (cp == 0x05F4 and !(Script(point_before[0]) == .Hebrew)) return error.IDNAFailure;
393 if (cp == 0x05F4 and !(Script(ucd.unicode_data.data_code[point_before]) == .Hebrew)) return error.IDNAFailure;
389394 // Appendix A.7. KATAKANA MIDDLE DOT
390395 if (false) return error.IDNAFailure;
391396 // Appendix A.8. ARABIC-INDIC DIGITS
......@@ -397,24 +402,24 @@ fn Validity_Criteria(
397402 point_before = p;
398403
399404 if (CheckBidi) {
400 switch (first_p[4]) {
405 switch (first_bc) {
401406 .L => {
402 switch (p[4]) {
407 switch (p_bc) {
403408 .L, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,
404409 .NSM => {},
405410 else => return error.IDNAFailure,
406411 }
407412 },
408413 .R, .AL => {
409 switch (p[4]) {
414 switch (p_bc) {
410415 .R, .AL, .AN, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,
411416 .NSM => {},
412417 else => return error.IDNAFailure,
413418 }
414 if (p[4] == .EN and seen_an) return error.IDNAFailure;
415 if (p[4] == .EN) seen_en = true;
416 if (p[4] == .AN and seen_en) return error.IDNAFailure;
417 if (p[4] == .AN) seen_an = true;
419 if (p_bc == .EN and seen_an) return error.IDNAFailure;
420 if (p_bc == .EN) seen_en = true;
421 if (p_bc == .AN and seen_en) return error.IDNAFailure;
422 if (p_bc == .AN) seen_an = true;
418423 },
419424 else => unreachable,
420425 }
......@@ -424,7 +429,7 @@ fn Validity_Criteria(
424429 if (check_zwnj_state == true and zwnj_state != .right_joining) return error.IDNAFailure;
425430 }
426431 if (CheckBidi) {
427 switch (first_p[4]) {
432 switch (first_bc) {
428433 .L => {
429434 switch (last_bidi_class) {
430435 .L, .EN => {},
normalization.zig+27-26
......@@ -4,9 +4,8 @@ const std = @import("std");
44const extras = @import("extras");
55const ucd = @import("unicode-ucd");
66
7const ucd_data_soa = ucd.unicode_data.data_soa;
87const data_first_gap = blk: {
9 for (ucd_data_soa[0], 0..) |cp, i| {
8 for (ucd.unicode_data.data_code, 0..) |cp, i| {
109 if (cp != i) {
1110 break :blk i;
1211 }
......@@ -50,16 +49,16 @@ fn Decomposition(map: *extras.ManyArrayList(u8), kind: enum { canonical, compati
5049 while (n < map.lengths.items.len) : (n += 1) {
5150 const sl = map.items(n);
5251 const cp = std.unicode.utf8Decode(sl) catch unreachable;
53 if (std.sort.binarySearch(u21, ucd_data_soa[0], cp, extras.compareFnBasic(u21))) |j| {
54 if (ucd_data_soa[5][j] == .__none) continue;
55 if ((ucd_data_soa[5][j] == .__canonical) != (kind == .canonical)) continue;
52 if (std.sort.binarySearch(u21, &ucd.unicode_data.data_code, cp, extras.compareFnBasic(u21))) |j| {
53 if (ucd.unicode_data.data_decomp[j] == .__none) continue;
54 if ((ucd.unicode_data.data_decomp[j] == .__canonical) != (kind == .canonical)) continue;
5655 map.remove(n);
57 for (ucd_data_soa[6][j], 0..) |ktem, k| {
56 for (ucd.unicode_data.data_decomp_map[j], 0..) |ktem, k| {
5857 var buf: [4]u8 = undefined;
5958 const l = std.unicode.utf8Encode(ktem, &buf) catch unreachable;
6059 try map.insertAt(n + k, buf[0..l]);
6160 }
62 n += ucd_data_soa[6][j].len - 1;
61 n += ucd.unicode_data.data_decomp_map[j].len - 1;
6362 }
6463 }
6564 // sort non-starters with respect to canonical ordering
......@@ -116,16 +115,17 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
116115 const slc = map.items(1);
117116 const c = std.unicode.utf8Decode(slc) catch unreachable;
118117
119 for (&ucd.unicode_data.data) |d| {
120 if (d[5] != .__canonical) continue;
121 if (d[6].len != 2) continue;
122 if (d[6][0] != l) continue;
123 if (d[6][1] != c) continue;
124 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, d[0], extras.compareFnBasic(u21)) != null) continue;
125 if (d[3] != 0) continue;
118 const ud = ucd.unicode_data;
119 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map, &ud.data_ccc) |cp, dc, dm, ccc| {
120 if (dc != .__canonical) continue;
121 if (dm.len != 2) continue;
122 if (dm[0] != l) continue;
123 if (dm[1] != c) continue;
124 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cp, extras.compareFnBasic(u21)) != null) continue;
125 if (ccc != 0) continue;
126126
127127 var buf: [4]u8 = undefined;
128 const len = std.unicode.utf8Encode(d[0], &buf) catch unreachable;
128 const len = std.unicode.utf8Encode(cp, &buf) catch unreachable;
129129 try map.set(0, buf[0..len]);
130130 map.remove(1);
131131 break;
......@@ -145,16 +145,17 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
145145 l = std.unicode.utf8Decode(sll) catch unreachable;
146146 if (cpCCC(l) == 0) break;
147147 }
148 for (&ucd.unicode_data.data) |d| {
149 if (d[5] != .__canonical) continue;
150 if (d[6].len != 2) continue;
151 if (d[6][0] != l) continue;
152 if (d[6][1] != c) continue;
153 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, d[0], extras.compareFnBasic(u21)) != null) continue;
154 if (d[3] != 0) continue;
148 const ud = ucd.unicode_data;
149 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map, &ud.data_ccc) |cp, dc, dm, ccc| {
150 if (dc != .__canonical) continue;
151 if (dm.len != 2) continue;
152 if (dm[0] != l) continue;
153 if (dm[1] != c) continue;
154 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cp, extras.compareFnBasic(u21)) != null) continue;
155 if (ccc != 0) continue;
155156
156157 var buf: [4]u8 = undefined;
157 const len = std.unicode.utf8Encode(d[0], &buf) catch unreachable;
158 const len = std.unicode.utf8Encode(cp, &buf) catch unreachable;
158159 try map.set(j, buf[0..len]);
159160 map.remove(i);
160161 i -= 1;
......@@ -189,10 +190,10 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
189190
190191pub fn cpCCC(cp: u21) u8 {
191192 if (cp < data_first_gap) {
192 return ucd_data_soa[3][cp];
193 return ucd.unicode_data.data_ccc[cp];
193194 }
194 if (std.sort.binarySearch(u21, ucd_data_soa[0], cp, extras.compareFnBasic(u21))) |idx| {
195 return ucd_data_soa[3][idx];
195 if (std.sort.binarySearch(u21, &ucd.unicode_data.data_code, cp, extras.compareFnBasic(u21))) |idx| {
196 return ucd.unicode_data.data_ccc[idx];
196197 }
197198 // TODO: this happens to be correct but ucd.unicode_data needs to detect the block ranges
198199 return 0;