2"""Generate the normalization and label validity tables from UnicodeData.txt,
3CompositionExclusions.txt and DerivedJoiningType.txt, and pack them into
6Use the same Unicode version as the mapping table (scripts/idna_table.py).
8 python3 scripts/unicode_data_table.py # print a summary
9 python3 scripts/unicode_data_table.py --write # update src/table_blob.inc
13Normalization (src/normalization.cpp). Each table is indexed by cp >> 8 into
14rows of 256 entries; blocks without data share one all-zero row.
16 decomposition_index/block/data
17 Row entry = (offset << 2) | flags. The decomposition of cp is
18 data[entry(cp) >> 2 .. entry(cp + 1) >> 2), so rows have 257 columns.
19 The data holds the full (recursive) canonical decomposition, or the full
20 compatibility decomposition when only that exists. Flags: 1 = the
21 decomposition is compatibility-only; 2 = canonical, but the full
22 compatibility decomposition differs. data[0] is unused. Hangul syllables
23 are decomposed algorithmically and have no entries.
25 Canonical_Combining_Class.
26 composition_index/block/data
27 Row entry = index into data, which holds (second, composite) pairs
28 sorted by second for each first code point; [entry(cp), entry(cp + 1))
29 are the pairs starting with cp. Only primary composites (canonical
30 decompositions of length two, not Full_Composition_Exclusion) appear.
33Validity (src/validity.cpp).
36 General_Category Mn, Mc or Me ranges: a label must not begin with a mark.
37 dir_start/dir_final/dir_value
38 Bidi_Class ranges of assigned code points for the RFC 5893 Bidi rule.
39 Unassigned code points are rejected by the mapping table first and look
41 joining_start/joining_final/joining_value
42 Joining_Type ranges for the RFC 5892 ContextJ rule for U+200C. Code
43 points without a range are U (Non_Joining). The rule's Virama test
44 uses Canonical_Combining_Class 9 from the ccc tables.
46from __future__
import annotations
52from collections
import defaultdict
54UNICODE_VERSION =
"18.0.0"
55UCD_URL =
"https://www.unicode.org/Public/{version}/ucd/{name}"
57MAX_CODE_POINT = 0x10FFFF
58BLOCK_COUNT = (MAX_CODE_POINT + 1) >> 8
63COMBINING_CATEGORIES = {
"Mn",
"Mc",
"Me"}
67 "NONE",
"BN",
"CS",
"ES",
"ON",
"EN",
"L",
"R",
"NSM",
"AL",
"AN",
"ET",
68 "WS",
"RLO",
"LRO",
"PDF",
"RLE",
"RLI",
"FSI",
"PDI",
"LRI",
"B",
"S",
73JOINING_TYPES = [
"U",
"C",
"D",
"L",
"R",
"T"]
77 """Return a UCD file, downloading it once into the working directory."""
78 stem, ext = os.path.splitext(os.path.basename(name))
79 cached = f
"{stem}-{version}{ext}"
80 if not os.path.exists(cached):
81 url = UCD_URL.format(version=version, name=name)
82 with urllib.request.urlopen(url)
as response:
83 data = response.read()
84 with open(cached,
"wb")
as f:
86 with open(cached, encoding=
"utf-8")
as f:
91 """The UnicodeData.txt fields used here, keyed by code point."""
95 self.
ccc: dict[int, int] = {}
96 self.
bidi: dict[int, str] = {}
100 for line
in text.splitlines():
103 fields = line.split(
";")
104 cp = int(fields[0], 16)
106 if name.endswith(
", First>"):
110 if name.endswith(
", Last>"):
111 first, range_first = range_first,
None
112 for c
in range(first, cp + 1):
114 self.
ccc[c] = int(fields[3])
115 self.
bidi[c] = fields[4]
116 decomposition = fields[5].split()
117 if decomposition
and decomposition[0].startswith(
"<"):
118 mapping = [int(x, 16)
for x
in decomposition[1:]]
121 self.
canonical[cp] = [int(x, 16)
for x
in decomposition]
125 if mapping
is None and compatibility:
137 excluded: set[int] = set()
138 for line
in text.splitlines():
139 line = line.split(
"#", 1)[0].strip()
142 first, _, last = line.partition(
"..")
143 excluded.update(range(int(first, 16), int(last
or first, 16) + 1))
148 """Map code points to JOINING_TYPES indexes; U is left out."""
149 types: dict[int, int] = {}
150 for line
in text.splitlines():
151 line = line.split(
"#", 1)[0].strip()
154 code_points, _, value = (x.strip()
for x
in line.partition(
";"))
155 if value
not in JOINING_TYPES:
156 raise SystemExit(f
"unknown Joining_Type value {value}")
159 first, _, last = code_points.partition(
"..")
160 for cp
in range(int(first, 16), int(last
or first, 16) + 1):
161 types[cp] = JOINING_TYPES.index(value)
166 entries: list[int], cols: int, block_is_empty: list[bool]
167) -> tuple[list[int], list[int]]:
168 """Split per-code-point entries into an index and deduplicated rows.
170 Row b covers entries[b * 256 : b * 256 + cols]. Empty blocks get a zero
173 index: list[int] = []
175 row_ids: dict[tuple[int, ...], int] = {}
176 for block
in range(BLOCK_COUNT):
177 if block_is_empty[block]:
180 row = tuple(entries[block << 8 : (block << 8) + cols])
181 if row
not in row_ids:
182 row_ids[row] = len(row_ids)
184 index.append(row_ids[row])
185 if len(row_ids) > 256:
186 raise SystemExit(f
"{len(row_ids)} rows do not fit a uint8_t index")
192 entries = [0] * (MAX_CODE_POINT + 2)
193 has_data = [
False] * BLOCK_COUNT
194 for cp
in range(MAX_CODE_POINT + 1):
196 decomposition: list[int] = []
197 if HANGUL_SBASE <= cp < HANGUL_SBASE + HANGUL_SCOUNT:
199 elif cp
in ucd.canonical:
200 decomposition = ucd.full_decomposition(cp, compatibility=
False)
201 if ucd.full_decomposition(cp, compatibility=
True) != decomposition:
203 elif cp
in ucd.compatibility:
204 decomposition = ucd.full_decomposition(cp, compatibility=
True)
206 entries[cp] = (len(data) << 2) | flags
208 data.extend(decomposition)
209 has_data[cp >> 8] =
True
210 entries[MAX_CODE_POINT + 1] = len(data) << 2
211 if len(data) >= 1 << 14:
212 raise SystemExit(
"decomposition data does not fit 14-bit offsets")
215 "decomposition_index": index,
216 "decomposition_block": block,
217 "decomposition_data": data,
222 entries = [ucd.ccc.get(cp, 0)
for cp
in range(MAX_CODE_POINT + 1)]
224 not any(entries[block << 8 : (block + 1) << 8])
225 for block
in range(BLOCK_COUNT)
228 return {
"ccc_index": index,
"ccc_block": block}
232 ucd: UnicodeData, exclusions: set[int]
233) -> dict[str, list[int]]:
234 pairs: dict[int, list[tuple[int, int]]] = defaultdict(list)
235 for cp, mapping
in ucd.canonical.items():
239 or ucd.ccc.get(cp, 0) != 0
240 or ucd.ccc.get(mapping[0], 0) != 0
243 pairs[mapping[0]].append((mapping[1], cp))
245 entries = [0] * (MAX_CODE_POINT + 2)
246 has_data = [
False] * BLOCK_COUNT
247 for cp
in range(MAX_CODE_POINT + 1):
248 entries[cp] = len(data)
249 for second, composite
in sorted(pairs.get(cp, ())):
250 data.extend((second, composite))
251 has_data[cp >> 8] =
True
252 entries[MAX_CODE_POINT + 1] = len(data)
253 if len(data) > 0xFFFF:
254 raise SystemExit(
"composition data does not fit 16-bit indexes")
257 "composition_index": index,
258 "composition_block": block,
259 "composition_data": data,
264 """Merge code points with equal values into (first, last, value) ranges."""
265 ranges: list[tuple[int, int, int]] = []
266 for cp
in sorted(values):
268 if ranges
and ranges[-1][2] == value
and ranges[-1][1] + 1 == cp:
269 ranges[-1] = (ranges[-1][0], cp, value)
271 ranges.append((cp, cp, value))
276 ucd: UnicodeData, joining_types: dict[int, int]
277) -> dict[str, list[int]]:
279 cp: 1
for cp, gc
in ucd.category.items()
if gc
in COMBINING_CATEGORIES
282 unknown = set(ucd.bidi.values()) - set(BIDI_CLASSES[1:])
284 raise SystemExit(f
"unknown Bidi_Class values {sorted(unknown)}")
286 {cp: BIDI_CLASSES.index(bidi)
for cp, bidi
in ucd.bidi.items()}
290 "combining_flat": [cp
for r
in combining
for cp
in r[:2]],
291 "dir_start": [first
for first, _, _
in directions],
292 "dir_final": [last
for _, last, _
in directions],
293 "dir_value": [value
for _, _, value
in directions],
294 "joining_start": [first
for first, _, _
in joining],
295 "joining_final": [last
for _, last, _
in joining],
296 "joining_value": [value
for _, _, value
in joining],
301 unicode_data_text: str,
302 composition_exclusions_text: str,
303 joining_type_text: str,
304) -> dict[str, list[int]]:
308 sections: dict[str, list[int]] = {}
317 """Return a UCD file whose '# Name-version.txt' header matches version."""
319 stem = os.path.splitext(os.path.basename(name))[0]
320 m = re.search(rf
"# {stem}-(\S+)\.txt", text)
321 if not m
or m.group(1) != version:
322 raise SystemExit(f
"{name} is not version {version}")
326def generate(version: str = UNICODE_VERSION) -> dict[str, list[int]]:
335 script_dir = os.path.dirname(os.path.abspath(__file__))
336 if script_dir
not in sys.path:
337 sys.path.insert(0, script_dir)
341 sections.update(generated)
345def main(argv: list[str]) -> int:
346 if argv
in ([
"-h"], [
"--help"]):
349 if argv
not in ([], [
"--write"], [
"-w"]):
354 print(f
"Unicode {UNICODE_VERSION}")
355 print(f
" decomposition: {len(g['decomposition_block']) // 257} rows, "
356 f
"{len(g['decomposition_data'])} code points")
357 print(f
" ccc: {len(g['ccc_block']) // 256} rows")
358 print(f
" composition: {len(g['composition_block']) // 257} rows, "
359 f
"{(len(g['composition_data']) - 1) // 2} pairs")
360 print(f
" combining: {len(g['combining_flat']) // 2} ranges")
361 print(f
" directions: {len(g['dir_start'])} ranges")
362 print(f
" joining types: {len(g['joining_start'])} ranges")
365 print(
"Normalization and validity tables packed into table_blob.inc")
369if __name__ ==
"__main__":
370 raise SystemExit(
main(sys.argv[1:]))
list[int] full_decomposition(self, int cp, bool compatibility)
None write_blob(dict[str, Any] sections, Path path=BLOB_PATH)
dict[str, list[int]] load_blob_sections(Path path=BLOB_PATH)
dict[str, list[int]] build_validity(UnicodeData ucd, dict[int, int] joining_types)
set[int] parse_composition_exclusions(str text)
dict[str, list[int]] generate(str version=UNICODE_VERSION)
dict[str, list[int]] build_composition(UnicodeData ucd, set[int] exclusions)
list[tuple[int, int, int]] build_ranges(dict[int, int] values)
str get_ucd_file(str name, str version=UNICODE_VERSION)
str get_versioned_ucd_file(str name, str version)
dict[str, list[int]] build_decomposition(UnicodeData ucd)
tuple[list[int], list[int]] build_multistage(list[int] entries, int cols, list[bool] block_is_empty)
None update_in_blob(dict[str, list[int]] generated)
dict[str, list[int]] build_sections(str unicode_data_text, str composition_exclusions_text, str joining_type_text)
dict[int, int] parse_joining_types(str text)
dict[str, list[int]] build_ccc(UnicodeData ucd)