1//! https://unicode.org/reports/tr15/
2
3const std = @import("std");
4const extras = @import("extras");
5const ucd = @import("unicode-ucd");
6
7const data_first_gap = blk: {
8 for (ucd.unicode_data.data_code, 0..) |cp, i| {
9 if (cp != i) {
10 break :blk i;
11 }
12 }
13};
14// comptime { @compileLog(data_first_gap); } // 888 as of 17.
15
16// hangul syllable constants
17const SBase = 0xAC00;
18const LBase = 0x1100;
19const VBase = 0x1161;
20const TBase = 0x11A7;
21const LCount = 19;
22const VCount = 21;
23const TCount = 28;
24const NCount = VCount * TCount;
25const SCount = LCount * NCount;
26
27pub fn ToNFD(map: *extras.ManyArrayList(u8)) !void {
28 try Decomposition(map, .canonical);
29}
30
31pub fn ToNFC(map: *extras.ManyArrayList(u8)) !void {
32 try ToNFD(map);
33 try CanonicalComposition(map);
34}
35
36pub fn ToNFKD(map: *extras.ManyArrayList(u8)) !void {
37 try Decomposition(map, .compatibility);
38}
39
40pub fn ToNFKC(map: *extras.ManyArrayList(u8)) !void {
41 try ToNFKD(map);
42 try CanonicalComposition(map);
43}
44
45// https://www.unicode.org/versions/latest/core-spec/chapter-3/#G733
46fn Decomposition(map: *extras.ManyArrayList(u8), kind: enum { canonical, compatibility }) !void {
47 // fully decompose all codepoints in the string
48 var n: usize = 0;
49 while (n < map.lengths.items.len) : (n += 1) {
50 const sl = map.items(n);
51 const cp = std.unicode.utf8Decode(sl) catch unreachable;
52 if (std.sort.binarySearch(u21, &ucd.unicode_data.data_code, cp, extras.compareFnBasic(u21))) |j| {
53 if (ucd.unicode_data.data_decomp[j] == .__none) continue;
54 if ((ucd.unicode_data.data_decomp[j] == .__canonical) != (kind == .canonical)) continue;
55 map.remove(n);
56 for (ucd.unicode_data.data_decomp_map[j], 0..) |ktem, k| {
57 var buf: [4]u8 = undefined;
58 const l = std.unicode.utf8Encode(ktem, &buf) catch unreachable;
59 try map.insertAt(n + k, buf[0..l]);
60 }
61 n += ucd.unicode_data.data_decomp_map[j].len - 1;
62 }
63 }
64 // sort non-starters with respect to canonical ordering
65 n = 0;
66 while (n < map.lengths.items.len) : (n += 1) {
67 const sln = map.items(n);
68 const cp = std.unicode.utf8Decode(sln) catch unreachable;
69 const ccc = cpCCC(cp);
70 if (ccc == 0) continue; // starter copdepoint
71 if (n == map.lengths.items.len - 1) break;
72 var buf: [128]u8 = @splat(0);
73 var len: u8 = 0;
74 buf[0] = ccc;
75 len += 1;
76 for (n + 1..map.lengths.items.len) |j| {
77 const slj = map.items(j);
78 const dp = std.unicode.utf8Decode(slj) catch unreachable;
79 const ddd = cpCCC(dp);
80 if (ddd == 0) break;
81 buf[len] = ddd;
82 len += 1;
83 }
84 const S = struct {
85 nstart: usize,
86 buf: []u8,
87 map: *extras.ManyArrayList(u8),
88
89 pub fn lessThan(self: *const @This(), a: usize, b: usize) bool {
90 return self.buf[a] < self.buf[b];
91 }
92 pub fn swap(self: *const @This(), a: usize, b: usize) void {
93 std.mem.swap(u8, &self.buf[a], &self.buf[b]);
94 self.map.swap(self.nstart + a, self.nstart + b);
95 }
96 };
97 std.mem.sortContext(0, len, S{
98 .nstart = n,
99 .buf = buf[0..len],
100 .map = map,
101 });
102 n += len;
103 }
104}
105
106// https://www.unicode.org/versions/latest/core-spec/chapter-3/#G50628
107fn CanonicalComposition(map: *extras.ManyArrayList(u8)) !void {
108 if (map.lengths.items.len < 2) {
109 return;
110 }
111
112 if (map.lengths.items.len == 2) {
113 const sll = map.items(0);
114 const l = std.unicode.utf8Decode(sll) catch unreachable;
115 const slc = map.items(1);
116 const c = std.unicode.utf8Decode(slc) catch unreachable;
117
118 const ud = ucd.unicode_data;
119 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map, &ud.data_ccc) |cp, dc, dm, ccc| {
120 if (dc != .__canonical) continue;
121 if (dm.len != 2) continue;
122 if (dm[0] != l) continue;
123 if (dm[1] != c) continue;
124 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cp, extras.compareFnBasic(u21)) != null) continue;
125 if (ccc != 0) continue;
126
127 var buf: [4]u8 = undefined;
128 const len = std.unicode.utf8Encode(cp, &buf) catch unreachable;
129 try map.set(0, buf[0..len]);
130 map.remove(1);
131 break;
132 }
133 return;
134 }
135
136 var i: usize = 1;
137 while (i < map.lengths.items.len) : (i += 1) {
138 const slc = map.items(i);
139 const c = std.unicode.utf8Decode(slc) catch unreachable;
140 var j: usize = i - 1;
141 var sll = map.items(j);
142 var l = std.unicode.utf8Decode(sll) catch unreachable;
143 while (j > 0) : (j -= 1) {
144 sll = map.items(j);
145 l = std.unicode.utf8Decode(sll) catch unreachable;
146 if (cpCCC(l) == 0) break;
147 }
148 const ud = ucd.unicode_data;
149 for (&ud.data_code, &ud.data_decomp, &ud.data_decomp_map, &ud.data_ccc) |cp, dc, dm, ccc| {
150 if (dc != .__canonical) continue;
151 if (dm.len != 2) continue;
152 if (dm[0] != l) continue;
153 if (dm[1] != c) continue;
154 if (std.sort.binarySearch(u21, &ucd.composition_exclusions.data, cp, extras.compareFnBasic(u21)) != null) continue;
155 if (ccc != 0) continue;
156
157 var buf: [4]u8 = undefined;
158 const len = std.unicode.utf8Encode(cp, &buf) catch unreachable;
159 try map.set(j, buf[0..len]);
160 map.remove(i);
161 i -= 1;
162 break;
163 }
164 }
165 // https://www.unicode.org/versions/latest/core-spec/chapter-3/#G56669
166 i = 0;
167 while (i < map.lengths.items.len) : (i += 1) {
168 const sll = map.items(i);
169 const cpl = std.unicode.utf8Decode(sll) catch unreachable;
170 if ((hangul_syllable_type(cpl) orelse continue) != .L) continue;
171
172 if (i + 1 == map.lengths.items.len) break;
173 const slv = map.items(i + 1);
174 const cpv = std.unicode.utf8Decode(slv) catch unreachable;
175 if ((hangul_syllable_type(cpv) orelse continue) != .V) continue;
176
177 if (i + 2 == map.lengths.items.len) {
178 //seq is only <L,V>
179 try composeHangulSyllable2(map, i, cpl, cpv);
180 continue;
181 }
182
183 const slt = map.items(i + 2);
184 const cpt = std.unicode.utf8Decode(slt) catch unreachable;
185 if ((hangul_syllable_type(cpt) orelse continue) != .T) continue;
186 // seq is <L,V,T>
187 try composeHangulSyllable3(map, i, cpl, cpv, cpt);
188 }
189}
190
191pub fn cpCCC(cp: u21) u8 {
192 if (cp < data_first_gap) {
193 return ucd.unicode_data.data_ccc[cp];
194 }
195 if (std.sort.binarySearch(u21, &ucd.unicode_data.data_code, cp, extras.compareFnBasic(u21))) |idx| {
196 return ucd.unicode_data.data_ccc[idx];
197 }
198 // TODO: this happens to be correct but ucd.unicode_data needs to detect the block ranges
199 return 0;
200}
201
202fn hangul_syllable_type(cp: u21) ?ucd.HangulSyllableType {
203 // can't use std.sort.binarySearch on this set because it's not sorted
204 // TODO: update ucd to have that property
205 for (&ucd.hangul_syllable_type.data) |d| {
206 if (cp >= d.from and cp <= d.to) {
207 return d.prop;
208 }
209 }
210 return null;
211}
212
213fn composeHangulSyllable2(map: *extras.ManyArrayList(u8), i: usize, cpl: u21, cpv: u21) !void {
214 const LIndex = cpl - LBase;
215 const VIndex = cpv - VBase;
216 const LVIndex = LIndex * NCount + VIndex * TCount;
217 const cps = SBase + LVIndex;
218 var buf: [4]u8 = undefined;
219 const len = std.unicode.utf8Encode(cps, &buf) catch unreachable;
220 try map.set(i, buf[0..len]);
221 map.remove(i + 1);
222}
223
224fn composeHangulSyllable3(map: *extras.ManyArrayList(u8), i: usize, cpl: u21, cpv: u21, cpt: u21) !void {
225 const LIndex = cpl - LBase;
226 const VIndex = cpv - VBase;
227 const TIndex = cpt - TBase;
228 const LVIndex = LIndex * NCount + VIndex * TCount;
229 const cps = SBase + LVIndex + TIndex;
230 var buf: [4]u8 = undefined;
231 const len = std.unicode.utf8Encode(cps, &buf) catch unreachable;
232 try map.set(i, buf[0..len]);
233 map.remove(i + 1);
234 map.remove(i + 1);
235}