Ada 4.1.1
Fast spec-compliant URL parser
Loading...
Searching...
No Matches
derived_table.py
Go to the documentation of this file.
1import os.path
2import requests
3import re
4import textwrap
5import sys
6if sys.version_info[0] < 3:
7 print('You need to run this with Python 3')
8 sys.exit(1)
9
10indent = ' ' * 4
11
12def fill(text):
13 tmp = textwrap.fill(text)
14 return textwrap.indent(tmp, indent)
15
16def filltab(text):
17 tmp = textwrap.fill(text, width=120)
18 return textwrap.indent(tmp, '\t')
19
21 return '{%s}' % (', '.join(map(str, arr)))
22
24 alllines = '{\n'
25 current_line = '\t'
26 for i in arr:
27 if(len(current_line) > 80):
28 if(current_line[-1] == ' '):
29 current_line = current_line[:-1]
30 alllines += current_line + '\n'
31 current_line = '\t'
32 current_line += str(i)
33 if(i != len(arr)-1): current_line += ', '
34 if(len(current_line) > 0):
35 if(current_line[-1] == ' '):
36 current_line = current_line[:-1]
37 alllines += current_line + '\n'
38 alllines += '\n}'
39 return alllines
40
41def compose2(f, g):
42 return lambda x: f(g(x))
43
45 alllines = '{'
46 for i in range(0,len(arr), 4):
47 alllines += '\n\t' + ', '.join(map(cpp_array_initializer,arr[i:i+4]))
48 if(i + 4 < len(arr)): alllines += ','
49 alllines += '\n}'
50 return alllines
51
52
53# Keep in step with UNICODE_VERSION in scripts/idna_table.py.
54UNICODE_VERSION = "18.0.0"
55urlderived = f"https://www.unicode.org/Public/{UNICODE_VERSION}/ucd/DerivedCoreProperties.txt"
56derivedfilename = f"DerivedCoreProperties-{UNICODE_VERSION}.txt"
58 if(not os.path.exists(derivedfilename)):
59 tablefile = requests.get(urlderived)
60 tablefile.raise_for_status()
61 with open(derivedfilename, 'wb') as file:
62 file.write(tablefile.content)
63 with open(derivedfilename, 'r') as file:
64 table_data = file.read()
65 version = get_version(table_data)
66 if version != UNICODE_VERSION:
67 raise SystemExit(f"{derivedfilename} is version {version}, expected "
68 f"{UNICODE_VERSION}; delete it to download again")
69 return table_data
70
71def get_version(table_data):
72 return re.search(r"# DerivedCoreProperties-(.*)\.txt", table_data).group(1)
73
74
76 start_pattern = re.compile(r'# Derived Property: ID_Start')
77 end_pattern = re.compile(r'# Derived Property: ID_Continue')
78
79 # Find the start and end indices of the section
80 start = start_pattern.search(text)
81 end = end_pattern.search(text)
82
83 if start is None or end is None:
84 raise ValueError("Section markers not found in the text.")
85
86 # Extract the relevant section
87 section = text[start.end():end.start()].strip().split('\n')
88 result = []
89 for line in section:
90 line = line.strip()
91 if not line or line.startswith('#'):
92 continue
93
94 parts = line.split(';')[0].strip().split('..')
95
96 # Convert hex to int and handle single value
97 if len(parts) == 1:
98 start_value = int(parts[0], 16)
99 result.append([start_value, start_value])
100 elif len(parts) == 2:
101 start_value = int(parts[0], 16)
102 end_value = int(parts[1], 16)
103 result.append([start_value, end_value])
104 else:
105 print(f"Warning: Line format not recognized: {line}")
106
107 return result
108
109
111 start_pattern = re.compile(r'# Derived Property: ID_Continue')
112 end_pattern = re.compile(r'# Derived Property: XID_Start')
113
114 # Find the start and end indices of the section
115 start = start_pattern.search(text)
116 end = end_pattern.search(text)
117
118 if start is None or end is None:
119 raise ValueError("Section markers not found in the text.")
120
121 # Extract the relevant section
122 section = text[start.end():end.start()].strip().split('\n')
123 result = []
124 for line in section:
125 line = line.strip()
126 if not line or line.startswith('#'):
127 continue
128
129 parts = line.split(';')[0].strip().split('..')
130
131 # Convert hex to int and handle single value
132 if len(parts) == 1:
133 start_value = int(parts[0], 16)
134 result.append([start_value, start_value])
135 elif len(parts) == 2:
136 start_value = int(parts[0], 16)
137 end_value = int(parts[1], 16)
138 result.append([start_value, end_value])
139 else:
140 print(f"Warning: Line format not recognized: {line}")
141
142 return result
143
145 """Build identifier range tables from DerivedCoreProperties.txt."""
146 table_data = get_derived_table()
147 version = get_version(table_data)
148 id_start = parse_id_start_section(table_data)
149 id_continue = parse_id_continue_section(table_data)
150 return {
151 "version": version,
152 "id_start": id_start,
153 "id_continue": id_continue,
154 }
155
156
158 """Print expanded C++ identifier tables to stdout (debug / legacy)."""
159 data = generate_derived()
160 print("// IDNA ", data["version"])
161 id_start = data["id_start"]
162 id_continue = data["id_continue"]
163 print("""
164// clang-format off
165#ifndef ADA_IDNA_IDENTIFIER_TABLES_H
166#define ADA_IDNA_IDENTIFIER_TABLES_H
167#include <cstdint>
168
169namespace ada::idna {
170""")
171 print("const uint32_t id_continue["+str(len(id_continue))+"][2] =")
172 print(cpp_arrayarray_initializer(id_continue), end=";\n")
173 print("const uint32_t id_start["+str(len(id_start))+"][2] =")
174 print(cpp_arrayarray_initializer(id_start), end=";\n")
175 print("""
176
177} // namespace ada::idna
178#endif // ADA_IDNA_IDENTIFIER_TABLES_H
179""")
180
181
183 """Write id_tables stub and update table_blob.inc with new ranges."""
184 script_dir = os.path.dirname(os.path.abspath(__file__))
185 if script_dir not in sys.path:
186 sys.path.insert(0, script_dir)
187 import pack_tables
188
189 data = generate_derived()
191 version=data["version"],
192 id_continue=data["id_continue"],
193 id_start=data["id_start"],
194 )
195 pack_tables.update_id_in_blob(data["id_continue"], data["id_start"])
196 print("Identifier tables regenerated and packed into table_blob.inc")
197
198
199if __name__ == "__main__":
200 if len(sys.argv) > 1 and sys.argv[1] in ("--write", "-w"):
202 elif len(sys.argv) > 1 and sys.argv[1] in ("-h", "--help"):
203 print("Usage: python3 scripts/derived_table.py [--write]")
204 print(" (default) Print expanded C++ identifier tables to stdout")
205 print(" --write Write stub + update src/table_blob.inc")
206 else:
multiline_cpp_array_initializer(arr)
cpp_array_initializer(arr)
cpp_arrayarray_initializer(arr)
parse_id_start_section(text)
parse_id_continue_section(text)
get_version(table_data)
None write_id_tables_stub(*, str version, list[list[int]] id_continue, list[list[int]] id_start, Path path=ID_CPP)
None update_id_in_blob(list[list[int]] id_continue, list[list[int]] id_start)