authorgravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2026-08-26 17:24:03-07:00
committergravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2026-08-26 17:24:03-07:00
log7b11277ab73bad88d8ec336ffae899f5c72ce07b
tree03687add7c0840e72d1a7eb88aa8e26de318f450
parent0f0a8e02e13be7d19ec182944070420e1598d8e0
signature Signed by SSH key SHA256:4hHJbtBRU58AYXwjL7fkz2fnQHdiye8x1QpTCQ0sHNw

don't use udc.unicode_data.data directly

doing so caused massive amounts of bloat (>6mb)

2 files changed, 67 insertions(+), 61 deletions(-)

idna.zig+40-35
...@@ -157,7 +157,7 @@ fn Processing(...@@ -157,7 +157,7 @@ fn Processing(
157 while (it.nextCodepointSlice()) |s| {157 while (it.nextCodepointSlice()) |s| {
158 const c = std.unicode.utf8Decode(s) catch unreachable;158 const c = std.unicode.utf8Decode(s) catch unreachable;
159 const p = ucd.unicode_data.find(c);159 const p = ucd.unicode_data.find(c);
160 switch (p[4]) {160 switch (ucd.unicode_data.data_bc[p]) {
161 .R, .AL, .AN => domain_is_bidi = true,161 .R, .AL, .AN => domain_is_bidi = true,
162 else => {},162 else => {},
163 }163 }
...@@ -229,11 +229,13 @@ fn Validity_Criteria(...@@ -229,11 +229,13 @@ fn Validity_Criteria(
229 const first_l = std.unicode.utf8ByteSequenceLength(first_b) catch unreachable;229 const first_l = std.unicode.utf8ByteSequenceLength(first_b) catch unreachable;
230 const first_c = std.unicode.utf8Decode(label[0..first_l]) catch unreachable;230 const first_c = std.unicode.utf8Decode(label[0..first_l]) catch unreachable;
231 const first_p = ucd.unicode_data.find(first_c);231 const first_p = ucd.unicode_data.find(first_c);
232 if (first_p[2] == .M) return error.IDNAFailure;232 const first_gc = ucd.unicode_data.data_gc[first_p];
233 if (first_p[2] == .Combining_Mark) return error.IDNAFailure;233 const first_bc = ucd.unicode_data.data_bc[first_p];
234 if (first_p[2] == .Mc) return error.IDNAFailure;234 if (first_gc == .M) return error.IDNAFailure;
235 if (first_p[2] == .Me) return error.IDNAFailure;235 if (first_gc == .Combining_Mark) return error.IDNAFailure;
236 if (first_p[2] == .Mn) return error.IDNAFailure;236 if (first_gc == .Mc) return error.IDNAFailure;
237 if (first_gc == .Me) return error.IDNAFailure;
238 if (first_gc == .Mn) return error.IDNAFailure;
237239
238 // 7. Each code point in the label must only have certain Status values according to Section 5, IDNA Mapping Table:240 // 7. Each code point in the label must only have certain Status values according to Section 5, IDNA Mapping Table:
239 var it = std.unicode.Utf8View.initUnchecked(label).iterator();241 var it = std.unicode.Utf8View.initUnchecked(label).iterator();
...@@ -241,8 +243,8 @@ fn Validity_Criteria(...@@ -241,8 +243,8 @@ fn Validity_Criteria(
241 var prev_cp: u21 = 0;243 var prev_cp: u21 = 0;
242244
243 // 8. If CheckJoiners, the label must satisify the ContextJ rules from RFC 5892 Appendix A.245 // 8. If CheckJoiners, the label must satisify the ContextJ rules from RFC 5892 Appendix A.
244 var point_2before: ucd.unicode_data.Codepoint = ucd.unicode_data.data[0];246 var point_2before: usize = 0;
245 var point_before: ucd.unicode_data.Codepoint = ucd.unicode_data.data[0];247 var point_before: usize = 0;
246 var zwnj_state: enum { start, dual_joining, transparent1, zwnj, transparent2, right_joining } = .start;248 var zwnj_state: enum { start, dual_joining, transparent1, zwnj, transparent2, right_joining } = .start;
247 var check_zwnj_state: ?bool = null;249 var check_zwnj_state: ?bool = null;
248250
...@@ -257,7 +259,7 @@ fn Validity_Criteria(...@@ -257,7 +259,7 @@ fn Validity_Criteria(
257 var seen_an = false;259 var seen_an = false;
258 var last_bidi_class: ucd.BidiClass = .NSM;260 var last_bidi_class: ucd.BidiClass = .NSM;
259 if (CheckBidi) {261 if (CheckBidi) {
260 switch (first_p[4]) {262 switch (first_bc) {
261 .L => {},263 .L => {},
262 .R, .AL => {},264 .R, .AL => {},
263 else => return error.IDNAFailure,265 else => return error.IDNAFailure,
...@@ -274,23 +276,25 @@ fn Validity_Criteria(...@@ -274,23 +276,25 @@ fn Validity_Criteria(
274 for (&ucd.derived_normalization_props.nfc_qc_n.data) |d| if (cp == d) return error.IDNAFailure;276 for (&ucd.derived_normalization_props.nfc_qc_n.data) |d| if (cp == d) return error.IDNAFailure;
275 for (&ucd.derived_normalization_props.nfc_qc_n.data_range) |d| if (cp >= d.from and cp <= d.to) return error.IDNAFailure;277 for (&ucd.derived_normalization_props.nfc_qc_n.data_range) |d| if (cp >= d.from and cp <= d.to) return error.IDNAFailure;
276 for (&ucd.derived_normalization_props.nfc_qc_m.data) |d| if (cp == d) {278 for (&ucd.derived_normalization_props.nfc_qc_m.data) |d| if (cp == d) {
277 for (&ucd.unicode_data.data) |r| {279 const ud = ucd.unicode_data;
278 if (r[5] != .__canonical) continue;280 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map) |_cp, dc, dm| {
279 if (r[6].len != 2) continue;281 if (dc != .__canonical) continue;
280 if (r[6][0] != prev_cp) continue;282 if (dm.len != 2) continue;
281 if (r[6][1] != cp) continue;283 if (dm[0] != prev_cp) continue;
282 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, r[0], extras.compareFnBasic(u21)) != null) continue;284 if (dm[1] != cp) continue;
285 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, _cp, extras.compareFnBasic(u21)) != null) continue;
283 // if (r[3] != 0) continue;286 // if (r[3] != 0) continue;
284 return error.IDNAFailure;287 return error.IDNAFailure;
285 }288 }
286 };289 };
287 for (&ucd.derived_normalization_props.nfc_qc_m.data_range) |d| if (cp >= d.from and cp <= d.to) {290 for (&ucd.derived_normalization_props.nfc_qc_m.data_range) |d| if (cp >= d.from and cp <= d.to) {
288 for (&ucd.unicode_data.data) |r| {291 const ud = ucd.unicode_data;
289 if (r[5] != .__canonical) continue;292 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map) |cpp, dc, dm| {
290 if (r[6].len != 2) continue;293 if (dc != .__canonical) continue;
291 if (r[6][0] != prev_cp) continue;294 if (dm.len != 2) continue;
292 if (r[6][1] != cp) continue;295 if (dm[0] != prev_cp) continue;
293 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, r[0], extras.compareFnBasic(u21)) != null) continue;296 if (dm[1] != cp) continue;
297 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cpp, extras.compareFnBasic(u21)) != null) continue;
294 // if (r[3] != 0) continue;298 // if (r[3] != 0) continue;
295 return error.IDNAFailure;299 return error.IDNAFailure;
296 }300 }
...@@ -316,12 +320,13 @@ fn Validity_Criteria(...@@ -316,12 +320,13 @@ fn Validity_Criteria(
316 }320 }
317321
318 const p = ucd.unicode_data.find(cp);322 const p = ucd.unicode_data.find(cp);
323 const p_bc = ucd.unicode_data.data_bc[p];
319324
320 if (CheckJoiners) {325 if (CheckJoiners) {
321 // Appendix A.1. ZERO WIDTH NON-JOINER326 // Appendix A.1. ZERO WIDTH NON-JOINER
322 if (cp == 0x200C) {327 if (cp == 0x200C) {
323 if (check_zwnj_state == null) check_zwnj_state = true;328 if (check_zwnj_state == null) check_zwnj_state = true;
324 if (point_before[3] == 9) check_zwnj_state = false;329 if (ucd.unicode_data.data_ccc[point_before] == 9) check_zwnj_state = false;
325 }330 }
326 switch (zwnj_state) {331 switch (zwnj_state) {
327 .start => {332 .start => {
...@@ -377,15 +382,15 @@ fn Validity_Criteria(...@@ -377,15 +382,15 @@ fn Validity_Criteria(
377 },382 },
378 }383 }
379 // Appendix A.2. ZERO WIDTH JOINER384 // Appendix A.2. ZERO WIDTH JOINER
380 if (cp == 0x200D and point_before[3] != 9) return error.IDNAFailure;385 if (cp == 0x200D and ucd.unicode_data.data_ccc[point_before] != 9) return error.IDNAFailure;
381 // Appendix A.3. MIDDLE DOT386 // Appendix A.3. MIDDLE DOT
382 if (point_before[0] == 0x00B7 and !(point_2before[0] == 0x006C and cp == 0x006C)) return error.IDNAFailure;387 if (ucd.unicode_data.data_code[point_before] == 0x00B7 and !(ucd.unicode_data.data_code[point_2before] == 0x006C and cp == 0x006C)) return error.IDNAFailure;
383 // Appendix A.4. GREEK LOWER NUMERAL SIGN (KERAIA)388 // Appendix A.4. GREEK LOWER NUMERAL SIGN (KERAIA)
384 if (point_before[0] == 0x0375 and !(Script(cp) == .Greek)) return error.IDNAFailure;389 if (ucd.unicode_data.data_code[point_before] == 0x0375 and !(Script(cp) == .Greek)) return error.IDNAFailure;
385 // Appendix A.5. HEBREW PUNCTUATION GERESH390 // Appendix A.5. HEBREW PUNCTUATION GERESH
386 if (cp == 0x05F3 and !(Script(point_before[0]) == .Hebrew)) return error.IDNAFailure;391 if (cp == 0x05F3 and !(Script(ucd.unicode_data.data_code[point_before]) == .Hebrew)) return error.IDNAFailure;
387 // Appendix A.6. HEBREW PUNCTUATION GERSHAYIM392 // Appendix A.6. HEBREW PUNCTUATION GERSHAYIM
388 if (cp == 0x05F4 and !(Script(point_before[0]) == .Hebrew)) return error.IDNAFailure;393 if (cp == 0x05F4 and !(Script(ucd.unicode_data.data_code[point_before]) == .Hebrew)) return error.IDNAFailure;
389 // Appendix A.7. KATAKANA MIDDLE DOT394 // Appendix A.7. KATAKANA MIDDLE DOT
390 if (false) return error.IDNAFailure;395 if (false) return error.IDNAFailure;
391 // Appendix A.8. ARABIC-INDIC DIGITS396 // Appendix A.8. ARABIC-INDIC DIGITS
...@@ -397,24 +402,24 @@ fn Validity_Criteria(...@@ -397,24 +402,24 @@ fn Validity_Criteria(
397 point_before = p;402 point_before = p;
398403
399 if (CheckBidi) {404 if (CheckBidi) {
400 switch (first_p[4]) {405 switch (first_bc) {
401 .L => {406 .L => {
402 switch (p[4]) {407 switch (p_bc) {
403 .L, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,408 .L, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,
404 .NSM => {},409 .NSM => {},
405 else => return error.IDNAFailure,410 else => return error.IDNAFailure,
406 }411 }
407 },412 },
408 .R, .AL => {413 .R, .AL => {
409 switch (p[4]) {414 switch (p_bc) {
410 .R, .AL, .AN, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,415 .R, .AL, .AN, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,
411 .NSM => {},416 .NSM => {},
412 else => return error.IDNAFailure,417 else => return error.IDNAFailure,
413 }418 }
414 if (p[4] == .EN and seen_an) return error.IDNAFailure;419 if (p_bc == .EN and seen_an) return error.IDNAFailure;
415 if (p[4] == .EN) seen_en = true;420 if (p_bc == .EN) seen_en = true;
416 if (p[4] == .AN and seen_en) return error.IDNAFailure;421 if (p_bc == .AN and seen_en) return error.IDNAFailure;
417 if (p[4] == .AN) seen_an = true;422 if (p_bc == .AN) seen_an = true;
418 },423 },
419 else => unreachable,424 else => unreachable,
420 }425 }
...@@ -424,7 +429,7 @@ fn Validity_Criteria(...@@ -424,7 +429,7 @@ fn Validity_Criteria(
424 if (check_zwnj_state == true and zwnj_state != .right_joining) return error.IDNAFailure;429 if (check_zwnj_state == true and zwnj_state != .right_joining) return error.IDNAFailure;
425 }430 }
426 if (CheckBidi) {431 if (CheckBidi) {
427 switch (first_p[4]) {432 switch (first_bc) {
428 .L => {433 .L => {
429 switch (last_bidi_class) {434 switch (last_bidi_class) {
430 .L, .EN => {},435 .L, .EN => {},
normalization.zig+27-26
...@@ -4,9 +4,8 @@ const std = @import("std");...@@ -4,9 +4,8 @@ const std = @import("std");
4const extras = @import("extras");4const extras = @import("extras");
5const ucd = @import("unicode-ucd");5const ucd = @import("unicode-ucd");
66
7const ucd_data_soa = ucd.unicode_data.data_soa;
8const data_first_gap = blk: {7const data_first_gap = blk: {
9 for (ucd_data_soa[0], 0..) |cp, i| {8 for (ucd.unicode_data.data_code, 0..) |cp, i| {
10 if (cp != i) {9 if (cp != i) {
11 break :blk i;10 break :blk i;
12 }11 }
...@@ -50,16 +49,16 @@ fn Decomposition(map: *extras.ManyArrayList(u8), kind: enum { canonical, compati...@@ -50,16 +49,16 @@ fn Decomposition(map: *extras.ManyArrayList(u8), kind: enum { canonical, compati
50 while (n < map.lengths.items.len) : (n += 1) {49 while (n < map.lengths.items.len) : (n += 1) {
51 const sl = map.items(n);50 const sl = map.items(n);
52 const cp = std.unicode.utf8Decode(sl) catch unreachable;51 const cp = std.unicode.utf8Decode(sl) catch unreachable;
53 if (std.sort.binarySearch(u21, ucd_data_soa[0], cp, extras.compareFnBasic(u21))) |j| {52 if (std.sort.binarySearch(u21, &ucd.unicode_data.data_code, cp, extras.compareFnBasic(u21))) |j| {
54 if (ucd_data_soa[5][j] == .__none) continue;53 if (ucd.unicode_data.data_decomp[j] == .__none) continue;
55 if ((ucd_data_soa[5][j] == .__canonical) != (kind == .canonical)) continue;54 if ((ucd.unicode_data.data_decomp[j] == .__canonical) != (kind == .canonical)) continue;
56 map.remove(n);55 map.remove(n);
57 for (ucd_data_soa[6][j], 0..) |ktem, k| {56 for (ucd.unicode_data.data_decomp_map[j], 0..) |ktem, k| {
58 var buf: [4]u8 = undefined;57 var buf: [4]u8 = undefined;
59 const l = std.unicode.utf8Encode(ktem, &buf) catch unreachable;58 const l = std.unicode.utf8Encode(ktem, &buf) catch unreachable;
60 try map.insertAt(n + k, buf[0..l]);59 try map.insertAt(n + k, buf[0..l]);
61 }60 }
62 n += ucd_data_soa[6][j].len - 1;61 n += ucd.unicode_data.data_decomp_map[j].len - 1;
63 }62 }
64 }63 }
65 // sort non-starters with respect to canonical ordering64 // sort non-starters with respect to canonical ordering
...@@ -116,16 +115,17 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {...@@ -116,16 +115,17 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
116 const slc = map.items(1);115 const slc = map.items(1);
117 const c = std.unicode.utf8Decode(slc) catch unreachable;116 const c = std.unicode.utf8Decode(slc) catch unreachable;
118117
119 for (&ucd.unicode_data.data) |d| {118 const ud = ucd.unicode_data;
120 if (d[5] != .__canonical) continue;119 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map, &ud.data_ccc) |cp, dc, dm, ccc| {
121 if (d[6].len != 2) continue;120 if (dc != .__canonical) continue;
122 if (d[6][0] != l) continue;121 if (dm.len != 2) continue;
123 if (d[6][1] != c) continue;122 if (dm[0] != l) continue;
124 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, d[0], extras.compareFnBasic(u21)) != null) continue;123 if (dm[1] != c) continue;
125 if (d[3] != 0) continue;124 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cp, extras.compareFnBasic(u21)) != null) continue;
125 if (ccc != 0) continue;
126126
127 var buf: [4]u8 = undefined;127 var buf: [4]u8 = undefined;
128 const len = std.unicode.utf8Encode(d[0], &buf) catch unreachable;128 const len = std.unicode.utf8Encode(cp, &buf) catch unreachable;
129 try map.set(0, buf[0..len]);129 try map.set(0, buf[0..len]);
130 map.remove(1);130 map.remove(1);
131 break;131 break;
...@@ -145,16 +145,17 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {...@@ -145,16 +145,17 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
145 l = std.unicode.utf8Decode(sll) catch unreachable;145 l = std.unicode.utf8Decode(sll) catch unreachable;
146 if (cpCCC(l) == 0) break;146 if (cpCCC(l) == 0) break;
147 }147 }
148 for (&ucd.unicode_data.data) |d| {148 const ud = ucd.unicode_data;
149 if (d[5] != .__canonical) continue;149 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map, &ud.data_ccc) |cp, dc, dm, ccc| {
150 if (d[6].len != 2) continue;150 if (dc != .__canonical) continue;
151 if (d[6][0] != l) continue;151 if (dm.len != 2) continue;
152 if (d[6][1] != c) continue;152 if (dm[0] != l) continue;
153 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, d[0], extras.compareFnBasic(u21)) != null) continue;153 if (dm[1] != c) continue;
154 if (d[3] != 0) continue;154 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cp, extras.compareFnBasic(u21)) != null) continue;
155 if (ccc != 0) continue;
155156
156 var buf: [4]u8 = undefined;157 var buf: [4]u8 = undefined;
157 const len = std.unicode.utf8Encode(d[0], &buf) catch unreachable;158 const len = std.unicode.utf8Encode(cp, &buf) catch unreachable;
158 try map.set(j, buf[0..len]);159 try map.set(j, buf[0..len]);
159 map.remove(i);160 map.remove(i);
160 i -= 1;161 i -= 1;
...@@ -189,10 +190,10 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {...@@ -189,10 +190,10 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
189190
190pub fn cpCCC(cp: u21) u8 {191pub fn cpCCC(cp: u21) u8 {
191 if (cp < data_first_gap) {192 if (cp < data_first_gap) {
192 return ucd_data_soa[3][cp];193 return ucd.unicode_data.data_ccc[cp];
193 }194 }
194 if (std.sort.binarySearch(u21, ucd_data_soa[0], cp, extras.compareFnBasic(u21))) |idx| {195 if (std.sort.binarySearch(u21, &ucd.unicode_data.data_code, cp, extras.compareFnBasic(u21))) |idx| {
195 return ucd_data_soa[3][idx];196 return ucd.unicode_data.data_ccc[idx];
196 }197 }
197 // TODO: this happens to be correct but ucd.unicode_data needs to detect the block ranges198 // TODO: this happens to be correct but ucd.unicode_data needs to detect the block ranges
198 return 0;199 return 0;