1const std = @import("std");
2const extras = @import("extras");
3const ucd = @import("unicode-ucd");
4const normalization = @import("./normalization.zig");
5const punycode = @import("./punycode.zig");
6
7pub const @"2008" = @import("./2008.zig");
8
9pub const table = @import("./table.zig");
10
11// https://unicode.org/reports/tr46/#ToASCII
12pub fn ToASCII(
13 allocator: std.mem.Allocator,
14 domain_name: []const u8,
15 CheckHyphens: bool,
16 CheckBidi: bool,
17 CheckJoiners: bool,
18 UseSTD3ASCIIRules: bool,
19 Transitional_Processing: bool, //deprecated
20 VerifyDnsLength: bool,
21 IgnoreInvalidPunycode: bool,
22) ![]u8 {
23 var map = extras.ManyArrayList(u8).init(allocator);
24 defer map.deinit();
25
26 try Processing(&map, domain_name, UseSTD3ASCIIRules, CheckHyphens, CheckBidi, CheckJoiners, Transitional_Processing, IgnoreInvalidPunycode);
27
28 map.lengths.clearRetainingCapacity();
29 var it2 = std.mem.splitScalar(u8, map.list.items, '.');
30 while (it2.next()) |label| {
31 try map.lengths.append(map.allocator, label.len);
32 try map.lengths.append(map.allocator, 1);
33 }
34 _ = map.lengths.orderedRemove(map.lengths.items.len - 1);
35 for (0..map.lengths.items.len) |n| {
36 punycode.encode(&map, n) catch |err| switch (err) {
37 error.InvalidPunycode => return error.IDNAFailure,
38 error.OutOfMemory => |e| return e,
39 };
40 }
41
42 if (VerifyDnsLength) {
43 // 1. The length of the domain name, excluding the root label and its dot, is from 1 to 253.
44 if (map.list.items.len < 1) return error.IDNAFailure;
45 if (map.list.items.len > 253) return error.IDNAFailure;
46 // 2. The length of each label is from 1 to 63.
47 for (map.lengths.items, 0..) |l, n| {
48 if (n % 2 == 1) continue;
49 if (l < 1 or l > 63) return error.IDNAFailure;
50 }
51 }
52
53 return map.toOwnedSlice();
54}
55
56// https://unicode.org/reports/tr46/#ToUnicode
57pub fn ToUnicode(
58 allocator: std.mem.Allocator,
59 domain_name: []const u8,
60 CheckHyphens: bool,
61 CheckBidi: bool,
62 CheckJoiners: bool,
63 UseSTD3ASCIIRules: bool,
64 Transitional_Processing: bool, //deprecated
65 IgnoreInvalidPunycode: bool,
66) ![]u8 {
67 var map = extras.ManyArrayList(u8).init(allocator);
68 defer map.deinit();
69 try Processing(&map, domain_name, UseSTD3ASCIIRules, CheckHyphens, CheckBidi, CheckJoiners, Transitional_Processing, IgnoreInvalidPunycode);
70 return map.toOwnedSlice();
71}
72
73// https://unicode.org/reports/tr46/#Processing
74fn Processing(
75 map: *extras.ManyArrayList(u8),
76 domain_name: []const u8,
77 UseSTD3ASCIIRules: bool,
78 CheckHyphens: bool,
79 CheckBidi: bool,
80 CheckJoiners: bool,
81 Transitional_Processing: bool, //deprecated
82 IgnoreInvalidPunycode: bool,
83) !void {
84 try map.list.ensureUnusedCapacity(map.allocator, domain_name.len);
85 try map.lengths.ensureUnusedCapacity(map.allocator, domain_name.len);
86 // 1. Map. For each code point in the domain_name string, look up the Status value in Section 5, IDNA Mapping Table, and take the following actions:
87 var it = std.unicode.Utf8View.initUnchecked(domain_name).iterator();
88 while (it.nextCodepointSlice()) |sl| {
89 const cp = std.unicode.utf8Decode(sl) catch unreachable;
90 const status, const mapping, const status2 = mappingRow(cp);
91 _ = status2;
92 sw: switch (status) {
93 .valid => {
94 try map.appendSlice(try map.add(), sl);
95 },
96 .ignored => {
97 //nop
98 },
99 .mapped => {
100 if (Transitional_Processing and cp == 'ẞ') {
101 try map.appendSlice(try map.add(), "s");
102 try map.appendSlice(try map.add(), "s");
103 continue;
104 }
105 for (mapping) |p| {
106 var buf: [4]u8 = undefined;
107 const l = std.unicode.utf8Encode(p, &buf) catch unreachable;
108 try map.appendSlice(try map.add(), buf[0..l]);
109 }
110 },
111 .deviation => {
112 continue :sw if (Transitional_Processing) .mapped else .valid;
113 },
114 .disallowed => {
115 continue :sw .valid;
116 },
117 }
118 }
119
120 // 2. Normalize. Normalize the domain_name string to Unicode Normalization Form C.
121 // https://unicode.org/reports/tr15/
122 if (!isNFC(map)) try normalization.ToNFC(map);
123
124 // 3. Break. Break the string into labels at U+002E ( . ) FULL STOP.
125 map.lengths.clearRetainingCapacity();
126 var it2 = std.mem.splitScalar(u8, map.list.items, '.');
127 while (it2.next()) |label| {
128 try map.lengths.append(map.allocator, label.len);
129 try map.lengths.append(map.allocator, 1);
130 }
131 _ = map.lengths.orderedRemove(map.lengths.items.len - 1);
132
133 // 4. Convert/Validate. For each label in the domain_name string:
134 for (0..map.lengths.items.len) |n| {
135 var label = map.items(n);
136 if (std.mem.eql(u8, label, ".")) continue;
137 if (std.mem.startsWith(u8, label, "xn--")) {
138 // 1. If the label contains any non-ASCII code point (i.e., a code point greater than U+007F), record that there was an error, and continue with the next label.
139 if (!extras.matchesAll(u8, label, std.ascii.isAscii)) return error.IDNAFailure;
140 // 2. Attempt to convert the rest of the label to Unicode according to Punycode [RFC3492]. If that conversion fails and if not IgnoreInvalidPunycode, record that there was an error, and continue with the next label. Otherwise replace the original label in the string by the results of the conversion.
141 punycode.decode(map, n) catch |err| switch (err) {
142 error.InvalidPunycode => if (!IgnoreInvalidPunycode) return error.IDNAFailure,
143 else => |e| return e,
144 };
145 label = map.items(n);
146 // 3. If the label is empty, or if the label contains only ASCII code points, record that there was an error.
147 if (label.len == 0) return error.IDNAFailure;
148 if (extras.matchesAll(u8, label, std.ascii.isAscii)) return error.IDNAFailure;
149 // 4. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for Nontransitional Processing. If any of the validity criteria are not satisfied, record that there was an error.
150 } else {
151 // 1. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for the input Processing choice (Transitional or Nontransitional). If any of the validity criteria are not satisfied, record that there was an error.
152 }
153 }
154
155 var domain_is_bidi = false;
156 it = std.unicode.Utf8View.initUnchecked(map.list.items).iterator();
157 while (it.nextCodepointSlice()) |s| {
158 const c = std.unicode.utf8Decode(s) catch unreachable;
159 const p = ucd.unicode_data.find(c);
160 switch (ucd.unicode_data.data_bc[p]) {
161 .R, .AL, .AN => domain_is_bidi = true,
162 else => {},
163 }
164 }
165
166 for (0..map.lengths.items.len) |n| {
167 const label = map.items(n);
168 if (std.mem.eql(u8, label, ".")) continue;
169
170 if (!extras.matchesAll(u8, label, std.ascii.isAscii)) {
171 // label was punycode converted above
172 // 4. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for Nontransitional Processing. If any of the validity criteria are not satisfied, record that there was an error.
173 try Validity_Criteria(label, CheckHyphens, false, UseSTD3ASCIIRules, CheckJoiners, CheckBidi and domain_is_bidi);
174 } else {
175 // label was already ascii
176 // 1. Verify that the label meets the validity criteria in Section 4.1, Validity Criteria for the input Processing choice (Transitional or Nontransitional). If any of the validity criteria are not satisfied, record that there was an error.
177 try Validity_Criteria(label, CheckHyphens, Transitional_Processing, UseSTD3ASCIIRules, CheckJoiners, CheckBidi and domain_is_bidi);
178 }
179 }
180}
181
182//
183//
184//
185
186fn mappingRow(cp: u21) struct { table.Status, []const u21, table.Status2008 } {
187 if (std.sort.binarySearch(table.Row, &table.data, cp, extras.compareFnField(u21, table.Row, .cp))) |idx| {
188 const row = table.data[idx];
189 return .{ row.status, row.mapping, row.status2 };
190 }
191 if (std.sort.binarySearch(table.RowRange, &table.data_range, cp, extras.compareFnRange(u21, table.RowRange, .from, .to))) |idx| {
192 const row = table.data_range[idx];
193 return .{ row.status, row.mapping, row.status2 };
194 }
195 unreachable;
196}
197
198// TODO:
199fn isNFC(map: *const extras.ManyArrayList(u8)) bool {
200 _ = map;
201 return false;
202}
203
204// https://unicode.org/reports/tr46/#Validity_Criteria
205fn Validity_Criteria(
206 label: []const u8,
207 CheckHyphens: bool,
208 Transitional_Processing: bool, //deprecated,
209 UseSTD3ASCIIRules: bool,
210 CheckJoiners: bool,
211 CheckBidi: bool,
212) !void {
213 // Each of the following criteria must be satisfied for a non-empty label:
214 if (label.len == 0) return;
215 // 1. The label must be in Unicode Normalization Form NFC.
216 {}
217 // 2. If CheckHyphens, the label must not contain a U+002D HYPHEN-MINUS character in both the third and fourth positions.
218 if (CheckHyphens and label.len >= 4 and label[2] == '-' and label[3] == '-') return error.IDNAFailure;
219 // 3. If CheckHyphens, the label must neither begin nor end with a U+002D HYPHEN-MINUS character.
220 if (CheckHyphens and std.mem.startsWith(u8, label, "-")) return error.IDNAFailure;
221 if (CheckHyphens and std.mem.endsWith(u8, label, "-")) return error.IDNAFailure;
222 // 4. If not CheckHyphens, the label must not begin with “xn--”.
223 if (!CheckHyphens and std.mem.startsWith(u8, label, "xn--")) return error.IDNAFailure;
224 // 5. The label must not contain a U+002E ( . ) FULL STOP.
225 if (std.mem.indexOfScalar(u8, label, '.') != null) return error.IDNAFailure;
226
227 // 6. The label must not begin with a combining mark, that is: General_Category=Mark.
228 const first_b = label[0];
229 const first_l = std.unicode.utf8ByteSequenceLength(first_b) catch unreachable;
230 const first_c = std.unicode.utf8Decode(label[0..first_l]) catch unreachable;
231 const first_p = ucd.unicode_data.find(first_c);
232 const first_gc = ucd.unicode_data.data_gc[first_p];
233 const first_bc = ucd.unicode_data.data_bc[first_p];
234 if (first_gc == .M) return error.IDNAFailure;
235 if (first_gc == .Combining_Mark) return error.IDNAFailure;
236 if (first_gc == .Mc) return error.IDNAFailure;
237 if (first_gc == .Me) return error.IDNAFailure;
238 if (first_gc == .Mn) return error.IDNAFailure;
239
240 // 7. Each code point in the label must only have certain Status values according to Section 5, IDNA Mapping Table:
241 var it = std.unicode.Utf8View.initUnchecked(label).iterator();
242 var prev_ccc: u8 = 0;
243 var prev_cp: u21 = 0;
244
245 // 8. If CheckJoiners, the label must satisify the ContextJ rules from RFC 5892 Appendix A.
246 var point_2before: usize = 0;
247 var point_before: usize = 0;
248 var zwnj_state: enum { start, dual_joining, transparent1, zwnj, transparent2, right_joining } = .start;
249 var check_zwnj_state: ?bool = null;
250
251 // 9. If CheckBidi, and if the domain name is a Bidi domain name, then the label must satisfy all six of the numbered conditions in RFC 5893 Section 2.
252 // 9.1. The first character must be a character with Bidi property L, R, or AL. If it has the R or AL property, it is an RTL label; if it has the L property, it is an LTR label.
253 // 9.2. In an RTL label, only characters with the Bidi properties R, AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
254 // 9.3. In an RTL label, the end of the label must be a character with Bidi property R, AL, EN, or AN, followed by zero or more characters with Bidi property NSM.
255 // 9.4. In an RTL label, if an EN is present, no AN may be present, and vice versa.
256 // 9.5. In an LTR label, only characters with the Bidi properties L, EN, ES, CS, ET, ON, BN, or NSM are allowed.
257 // 9.6. In an LTR label, the end of the label must be a character with Bidi property L or EN, followed by zero or more characters with Bidi property NSM.
258 var seen_en = false;
259 var seen_an = false;
260 var last_bidi_class: ucd.BidiClass = .NSM;
261 if (CheckBidi) {
262 switch (first_bc) {
263 .L => {},
264 .R, .AL => {},
265 else => return error.IDNAFailure,
266 }
267 }
268
269 while (it.nextCodepointSlice()) |sl| {
270 const cp = std.unicode.utf8Decode(sl) catch unreachable;
271 const status, _, _ = mappingRow(cp);
272
273 // [V1]
274 const ccc = normalization.cpCCC(cp);
275 if (prev_ccc > ccc and ccc > 0) return error.IDNAFailure;
276 for (&ucd.derived_normalization_props.nfc_qc_n.data) |d| if (cp == d) return error.IDNAFailure;
277 for (&ucd.derived_normalization_props.nfc_qc_n.data_range) |d| if (cp >= d.from and cp <= d.to) return error.IDNAFailure;
278 for (&ucd.derived_normalization_props.nfc_qc_m.data) |d| if (cp == d) {
279 const ud = ucd.unicode_data;
280 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map) |_cp, dc, dm| {
281 if (dc != .__canonical) continue;
282 if (dm.len != 2) continue;
283 if (dm[0] != prev_cp) continue;
284 if (dm[1] != cp) continue;
285 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, _cp, extras.compareFnBasic(u21)) != null) continue;
286 // if (r[3] != 0) continue;
287 return error.IDNAFailure;
288 }
289 };
290 for (&ucd.derived_normalization_props.nfc_qc_m.data_range) |d| if (cp >= d.from and cp <= d.to) {
291 const ud = ucd.unicode_data;
292 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map) |cpp, dc, dm| {
293 if (dc != .__canonical) continue;
294 if (dm.len != 2) continue;
295 if (dm[0] != prev_cp) continue;
296 if (dm[1] != cp) continue;
297 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cpp, extras.compareFnBasic(u21)) != null) continue;
298 // if (r[3] != 0) continue;
299 return error.IDNAFailure;
300 }
301 };
302 prev_ccc = ccc;
303 prev_cp = cp;
304
305 // 1. For Transitional Processing (deprecated), each value must be valid.
306 // 2. For Nontransitional Processing, each value must be either valid or deviation.
307 if (Transitional_Processing) {
308 if (!(status == .valid)) return error.IDNAFailure;
309 } else {
310 if (!(status == .valid or status == .deviation)) return error.IDNAFailure;
311 }
312
313 // 3. In addition, if UseSTD3ASCIIRules=true and the code point is an ASCII code point (U+0000..U+007F), then it must be a lowercase letter (a-z), a digit (0-9), or a hyphen-minus (U+002D).
314 // (Note: This excludes uppercase ASCII A-Z which are mapped in UTS #46 and disallowed in IDNA2008.)
315 if (UseSTD3ASCIIRules and std.ascii.isAscii(sl[0])) blk: {
316 if (std.ascii.isLower(sl[0])) break :blk;
317 if (std.ascii.isDigit(sl[0])) break :blk;
318 if (sl[0] == '-') break :blk;
319 return error.IDNAFailure;
320 }
321
322 const p = ucd.unicode_data.find(cp);
323 const p_bc = ucd.unicode_data.data_bc[p];
324
325 if (CheckJoiners) {
326 // Appendix A.1. ZERO WIDTH NON-JOINER
327 if (cp == 0x200C) {
328 if (check_zwnj_state == null) check_zwnj_state = true;
329 if (ucd.unicode_data.data_ccc[point_before] == 9) check_zwnj_state = false;
330 }
331 switch (zwnj_state) {
332 .start => {
333 if (Joining_Type(cp)) |t| {
334 if (t == .L) zwnj_state = .dual_joining;
335 if (t == .D) zwnj_state = .dual_joining;
336 }
337 },
338 .dual_joining => {
339 if (Joining_Type(cp)) |t| {
340 switch (t) {
341 .T => zwnj_state = .transparent1,
342 .L, .D => {},
343 else => zwnj_state = .transparent1,
344 }
345 } else {
346 zwnj_state = .transparent1;
347 }
348 if (cp == 0x200C) zwnj_state = .zwnj;
349 },
350 .transparent1 => {
351 if (cp == 0x200C) {
352 zwnj_state = .zwnj;
353 } else {
354 zwnj_state = .start;
355 }
356 },
357 .zwnj => {
358 if (Joining_Type(cp)) |t| {
359 switch (t) {
360 .T => zwnj_state = .transparent2,
361 .R, .D => zwnj_state = .right_joining,
362 .L => zwnj_state = .dual_joining,
363 else => zwnj_state = .transparent2,
364 }
365 } else {
366 zwnj_state = .transparent2;
367 }
368 },
369 .transparent2 => {
370 if (Joining_Type(cp)) |t| {
371 switch (t) {
372 .R, .D => zwnj_state = .right_joining,
373 .L => zwnj_state = .dual_joining,
374 else => zwnj_state = .start,
375 }
376 } else {
377 zwnj_state = .start;
378 }
379 },
380 .right_joining => {
381 //rule passed
382 },
383 }
384 // Appendix A.2. ZERO WIDTH JOINER
385 if (cp == 0x200D and ucd.unicode_data.data_ccc[point_before] != 9) return error.IDNAFailure;
386 // Appendix A.3. MIDDLE DOT
387 if (ucd.unicode_data.data_code[point_before] == 0x00B7 and !(ucd.unicode_data.data_code[point_2before] == 0x006C and cp == 0x006C)) return error.IDNAFailure;
388 // Appendix A.4. GREEK LOWER NUMERAL SIGN (KERAIA)
389 if (ucd.unicode_data.data_code[point_before] == 0x0375 and !(Script(cp) == .Greek)) return error.IDNAFailure;
390 // Appendix A.5. HEBREW PUNCTUATION GERESH
391 if (cp == 0x05F3 and !(Script(ucd.unicode_data.data_code[point_before]) == .Hebrew)) return error.IDNAFailure;
392 // Appendix A.6. HEBREW PUNCTUATION GERSHAYIM
393 if (cp == 0x05F4 and !(Script(ucd.unicode_data.data_code[point_before]) == .Hebrew)) return error.IDNAFailure;
394 // Appendix A.7. KATAKANA MIDDLE DOT
395 if (false) return error.IDNAFailure;
396 // Appendix A.8. ARABIC-INDIC DIGITS
397 if (false) return error.IDNAFailure;
398 // Appendix A.9. EXTENDED ARABIC-INDIC DIGITS
399 if (false) return error.IDNAFailure;
400 }
401 point_2before = point_before;
402 point_before = p;
403
404 if (CheckBidi) {
405 switch (first_bc) {
406 .L => {
407 switch (p_bc) {
408 .L, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,
409 .NSM => {},
410 else => return error.IDNAFailure,
411 }
412 },
413 .R, .AL => {
414 switch (p_bc) {
415 .R, .AL, .AN, .EN, .ES, .CS, .ET, .ON, .BN => |cl| last_bidi_class = cl,
416 .NSM => {},
417 else => return error.IDNAFailure,
418 }
419 if (p_bc == .EN and seen_an) return error.IDNAFailure;
420 if (p_bc == .EN) seen_en = true;
421 if (p_bc == .AN and seen_en) return error.IDNAFailure;
422 if (p_bc == .AN) seen_an = true;
423 },
424 else => unreachable,
425 }
426 }
427 }
428 if (CheckJoiners) {
429 if (check_zwnj_state == true and zwnj_state != .right_joining) return error.IDNAFailure;
430 }
431 if (CheckBidi) {
432 switch (first_bc) {
433 .L => {
434 switch (last_bidi_class) {
435 .L, .EN => {},
436 else => return error.IDNAFailure,
437 }
438 },
439 .R, .AL => {
440 switch (last_bidi_class) {
441 .R, .AL, .EN, .AN => {},
442 else => return error.IDNAFailure,
443 }
444 },
445 else => unreachable,
446 }
447 }
448}
449
450fn Script(cp: u21) ucd.ScriptLong {
451 for (&ucd.scripts.data) |d| {
452 if (cp >= d.from and cp <= d.to) {
453 return d.script;
454 }
455 }
456 unreachable;
457}
458fn Joining_Type(cp: u21) ?ucd.arabic_shaping.Joining.Type {
459 for (&ucd.arabic_shaping.data) |d| {
460 if (cp == d.codepoint) {
461 return d.joining_type;
462 }
463 }
464 return null;
465}