authorgravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2026-01-20 03:18:10 -08:00
committergravatar for hello@nektro.netMeghan Denny <hello@nektro.net> 2026-01-20 03:18:10 -08:00
log18bab1f29d39c90f00d6e6e9cd7a8a0a559ddb10
tree730795f9577db6fc65885cb00f62178d1515cfa9
parent85cc7a78d8a46bd5439aafef2449754877aa8d50

most of the fixes for toUnicodeFail


2 files changed, 99 insertions(+), 9 deletions(-)

idna.zig+99-6
...@@ -140,10 +140,34 @@ fn Processing(...@@ -140,10 +140,34 @@ fn Processing(
140 if (label.len == 0) return error.IDNAFailure;140 if (label.len == 0) return error.IDNAFailure;
141 if (extras.matchesAll(u8, label, std.ascii.isAscii)) return error.IDNAFailure;141 if (extras.matchesAll(u8, label, std.ascii.isAscii)) return error.IDNAFailure;
142 // 4. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for Nontransitional Processing. If any of the validity criteria are not satisfied, record that there was an error.142 // 4. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for Nontransitional Processing. If any of the validity criteria are not satisfied, record that there was an error.
143 try Validity_Criteria(label, CheckHyphens, true, UseSTD3ASCIIRules, CheckJoiners, CheckBidi);
144 } else {143 } else {
145 // 1. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for the input Processing choice (Transitional or Nontransitional). If any of the validity criteria are not satisfied, record that there was an error.144 // 1. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for the input Processing choice (Transitional or Nontransitional). If any of the validity criteria are not satisfied, record that there was an error.
146 try Validity_Criteria(label, CheckHyphens, Transitional_Processing, UseSTD3ASCIIRules, CheckJoiners, CheckBidi);145 }
146 }
147
148 var domain_is_bidi = false;
149 it = std.unicode.Utf8View.initUnchecked(map.list.items).iterator();
150 while (it.nextCodepointSlice()) |s| {
151 const c = std.unicode.utf8Decode(s) catch unreachable;
152 const p = ucd.unicode_data.find(c);
153 switch (p[4]) {
154 .R, .AL, .AN => domain_is_bidi = true,
155 else => {},
156 }
157 }
158
159 for (0..map.lengths.items.len) |n| {
160 const label = map.items(n);
161 if (std.mem.eql(u8, label, ".")) continue;
162
163 if (!extras.matchesAll(u8, label, std.ascii.isAscii)) {
164 // label was punycode converted above
165 // 4. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for Nontransitional Processing. If any of the validity criteria are not satisfied, record that there was an error.
166 try Validity_Criteria(label, CheckHyphens, false, UseSTD3ASCIIRules, CheckJoiners, CheckBidi and domain_is_bidi);
167 } else {
168 // label was already ascii
169 // 1. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for the input Processing choice (Transitional or Nontransitional). If any of the validity criteria are not satisfied, record that there was an error.
170 try Validity_Criteria(label, CheckHyphens, Transitional_Processing, UseSTD3ASCIIRules, CheckJoiners, CheckBidi and domain_is_bidi);
147 }171 }
148 }172 }
149}173}
...@@ -193,15 +217,84 @@ fn Validity_Criteria(...@@ -193,15 +217,84 @@ fn Validity_Criteria(
193 // 5. The label must not contain a U+002E ( . ) FULL STOP.217 // 5. The label must not contain a U+002E ( . ) FULL STOP.
194 if (std.mem.indexOfScalar(u8, label, '.') != null) return error.IDNAFailure;218 if (std.mem.indexOfScalar(u8, label, '.') != null) return error.IDNAFailure;
195219
196 _ = Transitional_Processing;
197 _ = UseSTD3ASCIIRules;
198 _ = CheckJoiners;
199 _ = CheckBidi;
200 // 6. The label must not begin with a combining mark, that is: General_Category=Mark.220 // 6. The label must not begin with a combining mark, that is: General_Category=Mark.
221 const first_b = label[0];
222 const first_l = std.unicode.utf8ByteSequenceLength(first_b) catch unreachable;
223 const first_c = std.unicode.utf8Decode(label[0..first_l]) catch unreachable;
224 const first_p = ucd.unicode_data.find(first_c);
225 if (first_p[2] == .M) return error.IDNAFailure;
226 if (first_p[2] == .Combining_Mark) return error.IDNAFailure;
227 if (first_p[2] == .Mc) return error.IDNAFailure;
228 if (first_p[2] == .Me) return error.IDNAFailure;
229 if (first_p[2] == .Mn) return error.IDNAFailure;
201230
202 // 7. Each code point in the label must only have certain Status values according to Section 5, IDNA Mapping Table:231 // 7. Each code point in the label must only have certain Status values according to Section 5, IDNA Mapping Table:
232 var it = std.unicode.Utf8View.initUnchecked(label).iterator();
233 while (it.nextCodepointSlice()) |sl| {
234 const cp = std.unicode.utf8Decode(sl) catch unreachable;
235 const status, _, _ = mappingRow(cp);
236
237 // 1. For Transitional Processing (deprecated), each value must be valid.
238 // 2. For Nontransitional Processing, each value must be either valid or deviation.
239 if (Transitional_Processing) {
240 if (!(status == .valid)) return error.IDNAFailure;
241 } else {
242 if (!(status == .valid or status == .deviation)) return error.IDNAFailure;
243 }
244
245 // 3. In addition, if UseSTD3ASCIIRules=true and the code point is an ASCII code point (U+0000..U+007F), then it must be a lowercase letter (a-z), a digit (0-9), or a hyphen-minus (U+002D).
246 // (Note: This excludes uppercase ASCII A-Z which are mapped in UTS #46 and disallowed in IDNA2008.)
247 if (UseSTD3ASCIIRules and std.ascii.isAscii(sl[0])) {
248 if (std.ascii.isLower(sl[0])) continue;
249 if (std.ascii.isDigit(sl[0])) continue;
250 if (sl[0] == '-') continue;
251 return error.IDNAFailure;
252 }
253 }
203254
204 // 8. If CheckJoiners, the label must satisify the ContextJ rules from RFC 5892 Appendix A.255 // 8. If CheckJoiners, the label must satisify the ContextJ rules from RFC 5892 Appendix A.
256 if (CheckJoiners) {
257 //
258 }
205259
206 // 9. If CheckBidi, and if the domain name is a Bidi domain name, then the label must satisfy all six of the numbered conditions in RFC 5893 Section 2.260 // 9. If CheckBidi, and if the domain name is a Bidi domain name, then the label must satisfy all six of the numbered conditions in RFC 5893 Section 2.
261 if (CheckBidi) {
262 // 1. The first character must be a character with Bidi property L, R, or AL. If it has the R or AL property, it is an RTL label; if it has the L property, it is an LTR label.
263 // 2. In an RTL label, only characters with the Bidi properties R, AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
264 // 3. In an RTL label, the end of the label must be a character with Bidi property R, AL, EN, or AN, followed by zero or more characters with Bidi property NSM.
265 // 4. In an RTL label, if an EN is present, no AN may be present, and vice versa.
266 // 5. In an LTR label, only characters with the Bidi properties L, EN, ES, CS, ET, ON, BN, or NSM are allowed.
267 // 6. In an LTR label, the end of the label must be a character with Bidi property L or EN, followed by zero or more characters with Bidi property NSM.
268 switch (first_p[4]) {
269 .L => {
270 it.i = 0;
271 while (it.nextCodepointSlice()) |sl| {
272 const cp = std.unicode.utf8Decode(sl) catch unreachable;
273 const point = ucd.unicode_data.find(cp);
274 switch (point[4]) {
275 .L, .EN, .ES, .CS, .ET, .ON, .BN, .NSM => {},
276 else => return error.IDNAFailure,
277 }
278 }
279 },
280 .R, .AL => {
281 it.i = 0;
282 var seen_en = false;
283 var seen_an = false;
284 while (it.nextCodepointSlice()) |sl| {
285 const cp = std.unicode.utf8Decode(sl) catch unreachable;
286 const point = ucd.unicode_data.find(cp);
287 switch (point[4]) {
288 .R, .AL, .AN, .EN, .ES, .CS, .ET, .ON, .BN, .NSM => {},
289 else => return error.IDNAFailure,
290 }
291 if (point[4] == .EN and seen_an) return error.IDNAFailure;
292 if (point[4] == .EN) seen_en = true;
293 if (point[4] == .AN and seen_en) return error.IDNAFailure;
294 if (point[4] == .AN) seen_an = true;
295 }
296 },
297 else => return error.IDNAFailure,
298 }
299 }
207}300}
normalization.zig-3
...@@ -165,13 +165,11 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {...@@ -165,13 +165,11 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
165 const sll = map.items(i);165 const sll = map.items(i);
166 const cpl = std.unicode.utf8Decode(sll) catch unreachable;166 const cpl = std.unicode.utf8Decode(sll) catch unreachable;
167 if ((hangul_syllable_type(cpl) orelse continue) != .L) continue;167 if ((hangul_syllable_type(cpl) orelse continue) != .L) continue;
168 std.debug.assert(cpl >= 0x1100 and cpl <= 0x1112);
169168
170 if (i + 1 == map.lengths.items.len) break;169 if (i + 1 == map.lengths.items.len) break;
171 const slv = map.items(i + 1);170 const slv = map.items(i + 1);
172 const cpv = std.unicode.utf8Decode(slv) catch unreachable;171 const cpv = std.unicode.utf8Decode(slv) catch unreachable;
173 if ((hangul_syllable_type(cpv) orelse continue) != .V) continue;172 if ((hangul_syllable_type(cpv) orelse continue) != .V) continue;
174 std.debug.assert(cpv >= 0x1161 and cpv <= 0x1175);
175173
176 if (i + 2 == map.lengths.items.len) {174 if (i + 2 == map.lengths.items.len) {
177 //seq is only <L,V>175 //seq is only <L,V>
...@@ -182,7 +180,6 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {...@@ -182,7 +180,6 @@ fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
182 const slt = map.items(i + 2);180 const slt = map.items(i + 2);
183 const cpt = std.unicode.utf8Decode(slt) catch unreachable;181 const cpt = std.unicode.utf8Decode(slt) catch unreachable;
184 if ((hangul_syllable_type(cpt) orelse continue) != .T) continue;182 if ((hangul_syllable_type(cpt) orelse continue) != .T) continue;
185 std.debug.assert(cpt >= 0x11A8 and cpt <= 0x11C2);
186 // seq is <L,V,T>183 // seq is <L,V,T>
187 try composeHangulSyllable3(map, i, cpl, cpv, cpt);184 try composeHangulSyllable3(map, i, cpl, cpv, cpt);
188 }185 }