Ada 4.1.1
Fast spec-compliant URL parser
Loading...
Searching...
No Matches
unicode.cpp
Go to the documentation of this file.
1#include "ada/unicode.h"
2
5#include "ada/common_defs.h"
6#include "ada/log.h"
7
9#include "ada_idna.cpp"
11
12#include <algorithm>
13#include <array>
14#include <cstdint>
15#include <cstring>
16#include <iterator>
17#include <optional>
18#include <string>
19#include <string_view>
20#include <utility>
21#if ADA_SSSE3
22#include <tmmintrin.h>
23#elif ADA_NEON
24#include <arm_neon.h>
25#elif ADA_SSE2
26#include <emmintrin.h>
27#elif ADA_LSX
28#include <lsxintrin.h>
29#elif ADA_RVV
30#include <riscv_vector.h>
31#endif
32
33#include <ranges>
34
35namespace ada::unicode {
36
37constexpr bool is_tabs_or_newline(char c) noexcept {
38 return c == '\r' || c == '\n' || c == '\t';
39}
40
41constexpr uint64_t broadcast(uint8_t v) noexcept {
42 return 0x101010101010101ull * v;
43}
44
45constexpr bool to_lower_ascii(char* input, size_t length) noexcept {
46 uint64_t broadcast_80 = broadcast(0x80);
47 uint64_t broadcast_Ap = broadcast(128 - 'A');
48 uint64_t broadcast_Zp = broadcast(128 - 'Z' - 1);
49 uint64_t non_ascii = 0;
50 size_t i = 0;
51
52 for (; i + 7 < length; i += 8) {
53 uint64_t word{};
54 memcpy(&word, input + i, sizeof(word));
55 non_ascii |= (word & broadcast_80);
56 word ^=
57 (((word + broadcast_Ap) ^ (word + broadcast_Zp)) & broadcast_80) >> 2;
58 memcpy(input + i, &word, sizeof(word));
59 }
60 if (i < length) {
61 uint64_t word{};
62 memcpy(&word, input + i, length - i);
63 non_ascii |= (word & broadcast_80);
64 word ^=
65 (((word + broadcast_Ap) ^ (word + broadcast_Zp)) & broadcast_80) >> 2;
66 memcpy(input + i, &word, length - i);
67 }
68 return non_ascii == 0;
69}
70#if ADA_SSSE3
71ada_really_inline bool has_tabs_or_newline(
72 std::string_view user_input) noexcept {
73 // first check for short strings in which case we do it naively.
74 if (user_input.size() < 16) { // slow path
75 return std::ranges::any_of(user_input, is_tabs_or_newline);
76 }
77 // fast path for long strings (expected to be common)
78 // Using SSSE3's _mm_shuffle_epi8 for table lookup (same approach as NEON)
79 size_t i = 0;
80 // Lookup table where positions 9, 10, 13 contain their own values
81 // Everything else is set to 1 so it won't match
82 const __m128i rnt =
83 _mm_setr_epi8(1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 0, 0, 13, 0, 0);
84 __m128i running = _mm_setzero_si128();
85 for (; i + 15 < user_input.size(); i += 16) {
86 __m128i word = _mm_loadu_si128((const __m128i*)(user_input.data() + i));
87 // Shuffle the lookup table using input bytes as indices
88 __m128i shuffled = _mm_shuffle_epi8(rnt, word);
89 // Compare: if shuffled value matches input, we found \t, \n, or \r
90 __m128i matches = _mm_cmpeq_epi8(shuffled, word);
91 running = _mm_or_si128(running, matches);
92 }
93 if (i < user_input.size()) {
94 __m128i word = _mm_loadu_si128(
95 (const __m128i*)(user_input.data() + user_input.length() - 16));
96 __m128i shuffled = _mm_shuffle_epi8(rnt, word);
97 __m128i matches = _mm_cmpeq_epi8(shuffled, word);
98 running = _mm_or_si128(running, matches);
99 }
100 return _mm_movemask_epi8(running) != 0;
101}
102#elif ADA_NEON
103ada_really_inline bool has_tabs_or_newline(
104 std::string_view user_input) noexcept {
105 // first check for short strings in which case we do it naively.
106 if (user_input.size() < 16) { // slow path
107 return std::ranges::any_of(user_input, is_tabs_or_newline);
108 }
109 // fast path for long strings (expected to be common)
110 size_t i = 0;
123 static uint8_t rnt_array[16] = {1, 0, 0, 0, 0, 0, 0, 0,
124 0, 9, 10, 0, 0, 13, 0, 0};
125 const uint8x16_t rnt = vld1q_u8(rnt_array);
126 // m['0xd', '0xa', '0x9']
127 uint8x16_t running{0};
128 for (; i + 15 < user_input.size(); i += 16) {
129 uint8x16_t word = vld1q_u8((const uint8_t*)user_input.data() + i);
130
131 running = vorrq_u8(running, vceqq_u8(vqtbl1q_u8(rnt, word), word));
132 }
133 if (i < user_input.size()) {
134 uint8x16_t word =
135 vld1q_u8((const uint8_t*)user_input.data() + user_input.length() - 16);
136 running = vorrq_u8(running, vceqq_u8(vqtbl1q_u8(rnt, word), word));
137 }
138 // `running` accumulates comparison results, so every lane is 0x00 or 0xFF:
139 // narrowing to four bits per lane and testing against zero is cheaper than a
140 // horizontal maximum. The double compare (ADA_NEON_SAFE_ZERO_CHECK set to 0)
141 // avoids a GPR transfer but is incorrect if flush-to-zero is enabled.
142 uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(running), 4);
143#if ADA_NEON_SAFE_ZERO_CHECK
144 return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0) != 0;
145#else
146 return vdupd_lane_f64(vreinterpret_f64_u8(narrowed), 0) != 0.0;
147#endif
148}
149#elif ADA_SSE2
150ada_really_inline bool has_tabs_or_newline(
151 std::string_view user_input) noexcept {
152 // first check for short strings in which case we do it naively.
153 if (user_input.size() < 16) { // slow path
154 return std::ranges::any_of(user_input, is_tabs_or_newline);
155 }
156 // fast path for long strings (expected to be common)
157 size_t i = 0;
158 const __m128i mask1 = _mm_set1_epi8('\r');
159 const __m128i mask2 = _mm_set1_epi8('\n');
160 const __m128i mask3 = _mm_set1_epi8('\t');
161 // If we supported SSSE3, we could use the algorithm that we use for NEON.
162 __m128i running{0};
163 for (; i + 15 < user_input.size(); i += 16) {
164 __m128i word = _mm_loadu_si128((const __m128i*)(user_input.data() + i));
165 running = _mm_or_si128(
166 _mm_or_si128(running, _mm_or_si128(_mm_cmpeq_epi8(word, mask1),
167 _mm_cmpeq_epi8(word, mask2))),
168 _mm_cmpeq_epi8(word, mask3));
169 }
170 if (i < user_input.size()) {
171 __m128i word = _mm_loadu_si128(
172 (const __m128i*)(user_input.data() + user_input.length() - 16));
173 running = _mm_or_si128(
174 _mm_or_si128(running, _mm_or_si128(_mm_cmpeq_epi8(word, mask1),
175 _mm_cmpeq_epi8(word, mask2))),
176 _mm_cmpeq_epi8(word, mask3));
177 }
178 return _mm_movemask_epi8(running) != 0;
179}
180#elif ADA_LSX
181ada_really_inline bool has_tabs_or_newline(
182 std::string_view user_input) noexcept {
183 // first check for short strings in which case we do it naively.
184 if (user_input.size() < 16) { // slow path
185 return std::ranges::any_of(user_input, is_tabs_or_newline);
186 }
187 // fast path for long strings (expected to be common)
188 size_t i = 0;
189 const __m128i mask1 = __lsx_vrepli_b('\r');
190 const __m128i mask2 = __lsx_vrepli_b('\n');
191 const __m128i mask3 = __lsx_vrepli_b('\t');
192 // If we supported SSSE3, we could use the algorithm that we use for NEON.
193 __m128i running{0};
194 for (; i + 15 < user_input.size(); i += 16) {
195 __m128i word = __lsx_vld((const __m128i*)(user_input.data() + i), 0);
196 running = __lsx_vor_v(
197 __lsx_vor_v(running, __lsx_vor_v(__lsx_vseq_b(word, mask1),
198 __lsx_vseq_b(word, mask2))),
199 __lsx_vseq_b(word, mask3));
200 }
201 if (i < user_input.size()) {
202 __m128i word = __lsx_vld(
203 (const __m128i*)(user_input.data() + user_input.length() - 16), 0);
204 running = __lsx_vor_v(
205 __lsx_vor_v(running, __lsx_vor_v(__lsx_vseq_b(word, mask1),
206 __lsx_vseq_b(word, mask2))),
207 __lsx_vseq_b(word, mask3));
208 }
209 if (__lsx_bz_v(running)) return false;
210 return true;
211}
212#elif ADA_RVV
213ada_really_inline bool has_tabs_or_newline(
214 std::string_view user_input) noexcept {
215 uint8_t* src = (uint8_t*)user_input.data();
216 for (size_t vl, n = user_input.size(); n > 0; n -= vl, src += vl) {
217 vl = __riscv_vsetvl_e8m1(n);
218 vuint8m1_t v = __riscv_vle8_v_u8m1(src, vl);
219 vbool8_t m1 = __riscv_vmseq(v, '\r', vl);
220 vbool8_t m2 = __riscv_vmseq(v, '\n', vl);
221 vbool8_t m3 = __riscv_vmseq(v, '\t', vl);
222 vbool8_t m = __riscv_vmor(__riscv_vmor(m1, m2, vl), m3, vl);
223 long idx = __riscv_vfirst(m, vl);
224 if (idx >= 0) return true;
225 }
226 return false;
227}
228#else
229ada_really_inline bool has_tabs_or_newline(
230 std::string_view user_input) noexcept {
231 auto has_zero_byte = [](uint64_t v) {
232 return ((v - 0x0101010101010101) & ~(v) & 0x8080808080808080);
233 };
234 size_t i = 0;
235 uint64_t mask1 = broadcast('\r');
236 uint64_t mask2 = broadcast('\n');
237 uint64_t mask3 = broadcast('\t');
238 uint64_t running{0};
239 for (; i + 7 < user_input.size(); i += 8) {
240 uint64_t word{};
241 memcpy(&word, user_input.data() + i, sizeof(word));
242 uint64_t xor1 = word ^ mask1;
243 uint64_t xor2 = word ^ mask2;
244 uint64_t xor3 = word ^ mask3;
245 running |= has_zero_byte(xor1) | has_zero_byte(xor2) | has_zero_byte(xor3);
246 }
247 if (i < user_input.size()) {
248 uint64_t word{};
249 memcpy(&word, user_input.data() + i, user_input.size() - i);
250 uint64_t xor1 = word ^ mask1;
251 uint64_t xor2 = word ^ mask2;
252 uint64_t xor3 = word ^ mask3;
253 running |= has_zero_byte(xor1) | has_zero_byte(xor2) | has_zero_byte(xor3);
254 }
255 return running;
256}
257#endif
258
259// A forbidden host code point is U+0000 NULL, U+0009 TAB, U+000A LF, U+000D CR,
260// U+0020 SPACE, U+0023 (#), U+002F (/), U+003A (:), U+003C (<), U+003E (>),
261// U+003F (?), U+0040 (@), U+005B ([), U+005C (\‍), U+005D (]), U+005E (^), or
262// U+007C (|).
263constexpr static std::array<uint8_t, 256> is_forbidden_host_code_point_table =
264 []() consteval {
265 std::array<uint8_t, 256> result{};
266 for (uint8_t c : {'\0', '\x09', '\x0a', '\x0d', ' ', '#', '/', ':', '<',
267 '>', '?', '@', '[', '\\', ']', '^', '|'}) {
268 result[c] = true;
269 }
270 return result;
271 }();
272
273ada_really_inline constexpr bool is_forbidden_host_code_point(
274 const char c) noexcept {
275 return is_forbidden_host_code_point_table[uint8_t(c)];
276}
277
278constexpr static std::array<uint8_t, 256> is_forbidden_domain_code_point_table =
279 []() consteval {
280 std::array<uint8_t, 256> result{};
281 for (uint8_t c : {'\0', '\x09', '\x0a', '\x0d', ' ', '#', '/', ':', '<',
282 '>', '?', '@', '[', '\\', ']', '^', '|', '%'}) {
283 result[c] = true;
284 }
285 for (uint8_t c = 0; c <= 32; c++) {
286 result[c] = true;
287 }
288 for (size_t c = 127; c < 256; c++) {
289 result[c] = true;
290 }
291 return result;
292 }();
293
294static_assert(sizeof(is_forbidden_domain_code_point_table) == 256);
295
296ada_really_inline constexpr bool is_forbidden_domain_code_point(
297 const char c) noexcept {
298 return is_forbidden_domain_code_point_table[uint8_t(c)];
299}
300
301ada_really_inline constexpr bool contains_forbidden_domain_code_point(
302 const char* input, size_t length) noexcept {
303 size_t i = 0;
304 uint8_t accumulator{};
305 for (; i + 4 <= length; i += 4) {
306 accumulator |= is_forbidden_domain_code_point_table[uint8_t(input[i])];
307 accumulator |= is_forbidden_domain_code_point_table[uint8_t(input[i + 1])];
308 accumulator |= is_forbidden_domain_code_point_table[uint8_t(input[i + 2])];
309 accumulator |= is_forbidden_domain_code_point_table[uint8_t(input[i + 3])];
310 }
311 for (; i < length; i++) {
312 accumulator |= is_forbidden_domain_code_point_table[uint8_t(input[i])];
313 }
314 return accumulator;
315}
316
317constexpr static std::array<uint8_t, 256>
319 std::array<uint8_t, 256> result{};
320 for (uint8_t c : {'\0', '\x09', '\x0a', '\x0d', ' ', '#', '/', ':', '<',
321 '>', '?', '@', '[', '\\', ']', '^', '|', '%'}) {
322 result[c] = 1;
323 }
324 for (uint8_t c = 'A'; c <= 'Z'; c++) {
325 result[c] = 2;
326 }
327 for (uint8_t c = 0; c <= 32; c++) {
328 result[c] = 1;
329 }
330 for (size_t c = 127; c < 256; c++) {
331 result[c] = 1;
332 }
333 return result;
334 }();
335
336ada_really_inline constexpr uint8_t
337contains_forbidden_domain_code_point_or_upper(const char* input,
338 size_t length) noexcept {
339 size_t i = 0;
340 uint8_t accumulator{};
341 for (; i + 4 <= length; i += 4) {
342 accumulator |=
344 accumulator |=
346 accumulator |=
348 accumulator |=
350 }
351 for (; i < length; i++) {
352 accumulator |=
354 }
355 return accumulator;
356}
357
358// std::isalnum(c) || c == '+' || c == '-' || c == '.') is true for
359constexpr static std::array<bool, 256> is_alnum_plus_table = []() consteval {
360 std::array<bool, 256> result{};
361 for (size_t c = 0; c < 256; c++) {
362 result[c] = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
363 (c >= 'A' && c <= 'Z') || c == '+' || c == '-' || c == '.';
364 }
365 return result;
366}();
367
368ada_really_inline constexpr bool is_alnum_plus(const char c) noexcept {
369 return is_alnum_plus_table[uint8_t(c)];
370 // A table is almost surely much faster than the
371 // following under most compilers: return
372 // return (std::isalnum(c) || c == '+' || c == '-' || c == '.');
373}
374
375ada_really_inline constexpr bool is_ascii_hex_digit(const char c) noexcept {
376 return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') ||
377 (c >= 'a' && c <= 'f');
378}
379
380ada_really_inline constexpr bool is_ascii_digit(const char c) noexcept {
381 // An ASCII digit is a code point in the range U+0030 (0) to U+0039 (9),
382 // inclusive.
383 return (c >= '0' && c <= '9');
384}
385
386ada_really_inline constexpr bool is_ascii(const char32_t c) noexcept {
387 // If code point is between U+0000 and U+007F inclusive, then return true.
388 return c <= 0x7F;
389}
390
391ada_really_inline constexpr bool is_c0_control_or_space(const char c) noexcept {
392 return (unsigned char)c <= ' ';
393}
394
395ada_really_inline constexpr bool is_ascii_tab_or_newline(
396 const char c) noexcept {
397 return c == '\t' || c == '\n' || c == '\r';
398}
399
400constexpr std::string_view table_is_double_dot_path_segment[] = {
401 "..", "%2e.", ".%2e", "%2e%2e"};
402
403ada_really_inline constexpr bool is_double_dot_path_segment(
404 std::string_view input) noexcept {
405 // This will catch most cases:
406 // The length must be 2,4 or 6.
407 // We divide by two and require
408 // that the result be between 1 and 3 inclusively.
409 uint64_t half_length = uint64_t(input.size()) / 2;
410 if (half_length - 1 > 2) {
411 return false;
412 }
413 // We have a string of length 2, 4 or 6.
414 // We now check the first character:
415 if ((input[0] != '.') && (input[0] != '%')) {
416 return false;
417 }
418 // We are unlikely the get beyond this point.
419 int hash_value = (input.size() + (unsigned)(input[0])) & 3;
420 const std::string_view target = table_is_double_dot_path_segment[hash_value];
421 if (target.size() != input.size()) {
422 return false;
423 }
424 // We almost never get here.
425 // Optimizing the rest is relatively unimportant.
426 auto prefix_equal_unsafe = [](std::string_view a, std::string_view b) {
427 uint16_t A, B;
428 memcpy(&A, a.data(), sizeof(A));
429 memcpy(&B, b.data(), sizeof(B));
430 return A == B;
431 };
432 if (!prefix_equal_unsafe(input, target)) {
433 return false;
434 }
435 for (size_t i = 2; i < input.size(); i++) {
436 char c = input[i];
437 if ((uint8_t((c | 0x20) - 0x61) <= 25 ? (c | 0x20) : c) != target[i]) {
438 return false;
439 }
440 }
441 return true;
442 // The above code might be a bit better than the code below. Compilers
443 // are not stupid and may use the fact that these strings have length 2,4 and
444 // 6 and other tricks.
445 // return input == ".." ||
446 // input == ".%2e" || input == ".%2E" ||
447 // input == "%2e." || input == "%2E." ||
448 // input == "%2e%2e" || input == "%2E%2E" || input == "%2E%2e" || input ==
449 // "%2e%2E";
450}
451
452ada_really_inline constexpr bool is_single_dot_path_segment(
453 std::string_view input) noexcept {
454 return input == "." || input == "%2e" || input == "%2E";
455}
456
457ada_really_inline constexpr bool is_lowercase_hex(const char c) noexcept {
458 return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f');
459}
460
461constexpr static char hex_to_binary_table[] = {
462 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 10, 11,
463 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
464 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 11, 12, 13, 14, 15};
465unsigned constexpr convert_hex_to_binary(const char c) noexcept {
466 return hex_to_binary_table[c - '0'];
467}
468
469std::string percent_decode(const std::string_view input, size_t first_percent) {
470 // next line is for safety only, we expect users to avoid calling
471 // percent_decode when first_percent is outside the range.
472 if (first_percent == std::string_view::npos) {
473 return std::string(input);
474 }
475
476 // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
477 const char* const src = input.data();
478 const char* const end = src + input.size();
479
480 // Decoding never grows the string, so a single pre-sized buffer written via
481 // bulk memcpy of the plain runs (then shrunk to the final length) avoids the
482 // byte-at-a-time appends of the naive version.
483 std::string out(input.size(), '\0');
484 char* d = out.data();
485 char* const d0 = d;
486
487 std::memcpy(d, src, first_percent);
488 d += first_percent;
489
490 const char* p = src + first_percent;
491 while (p < end) {
492 if (*p == '%') {
493 // Decode runs of valid %XX tightly (common for nested/encoded URLs).
494 while (p + 2 < end && *p == '%') {
495 if (!is_ascii_hex_digit(p[1]) || !is_ascii_hex_digit(p[2])) {
496 break;
497 }
498 *d++ = static_cast<char>(convert_hex_to_binary(p[1]) * 16 +
499 convert_hex_to_binary(p[2]));
500 p += 3;
501 }
502 if (p < end && *p == '%') {
503 // Not a valid escape (too few chars left or bad hex): copy '%'
504 // literally and keep scanning after it.
505 *d++ = *p++;
506 }
507 } else {
508 const char* q = static_cast<const char*>(
509 std::memchr(p, '%', static_cast<size_t>(end - p)));
510 const char* run_end = q ? q : end;
511 const size_t n = static_cast<size_t>(run_end - p);
512 std::memcpy(d, p, n);
513 d += n;
514 p = run_end;
515 }
516 }
517
518 out.resize(static_cast<size_t>(d - d0));
519 return out;
520}
521
522// 0..15 for hex digits, 0xFF otherwise - validate and decode with two loads.
523constexpr static std::array<uint8_t, 256> unhex_table = []() consteval {
524 std::array<uint8_t, 256> t{};
525 for (size_t i = 0; i < 256; ++i) {
526 t[i] = 0xFF;
527 }
528 for (uint8_t i = 0; i < 10; ++i) {
529 t[static_cast<size_t>('0') + i] = i;
530 }
531 for (uint8_t i = 0; i < 6; ++i) {
532 t[static_cast<size_t>('A') + i] = static_cast<uint8_t>(10 + i);
533 t[static_cast<size_t>('a') + i] = static_cast<uint8_t>(10 + i);
534 }
535 return t;
536}();
537
538std::string form_urlencoded_decode(const std::string_view input) {
539 const size_t len = input.size();
540 if (len == 0) [[unlikely]] {
541 return {};
542 }
543
544 // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
545 const char* const src = input.data();
546 const char* const end = src + len;
547 const char* p = src;
548
549 // Advance over the untransformed prefix.
550 while (p < end && *p != '+' && *p != '%') {
551 ++p;
552 }
553 if (p == end) {
554 return std::string(input);
555 }
556
557 // Output is always at most as long as the input: write into a single
558 // pre-sized buffer, then shrink to the final length.
559 std::string out(len, '\0');
560 char* d = out.data();
561 char* const d0 = d;
562
563 const size_t prefix = static_cast<size_t>(p - src);
564 std::memcpy(d, src, prefix);
565 d += prefix;
566
567 while (p < end) {
568 const char c = *p;
569 if (c == '+') {
570 *d++ = ' ';
571 ++p;
572 } else if (c == '%') {
573 // Decode runs of valid %XX tightly (common for nested URL query values).
574 while (p + 2 < end && *p == '%') {
575 const uint8_t hi = unhex_table[static_cast<uint8_t>(p[1])];
576 const uint8_t lo = unhex_table[static_cast<uint8_t>(p[2])];
577 if ((hi | lo) >= 16) {
578 break;
579 }
580 *d++ = static_cast<char>((hi << 4) | lo);
581 p += 3;
582 }
583 if (p < end && *p == '%') {
584 // Invalid escape: copy '%' literally and continue.
585 *d++ = *p++;
586 }
587 } else {
588 // Copy a plain run until the next '+' or '%'.
589 const char* start = p;
590 ++p;
591 while (p < end && *p != '+' && *p != '%') {
592 ++p;
593 }
594 const size_t n = static_cast<size_t>(p - start);
595 std::memcpy(d, start, n);
596 d += n;
597 }
598 }
599
600 out.resize(static_cast<size_t>(d - d0));
601 return out;
602}
603
604void percent_encode_suffix(const char* p, const char* end,
605 const uint8_t character_set[], std::string& out);
606
607std::string percent_encode(const std::string_view input,
608 const uint8_t character_set[]) {
609 auto pointer = std::ranges::find_if(input, [character_set](const char c) {
610 return character_sets::bit_at(character_set, c);
611 });
612 // Optimization: Don't iterate if percent encode is not required
613 if (pointer == input.end()) {
614 return std::string(input);
615 }
616
617 std::string result;
618 result.reserve(input.length()); // in the worst case, percent encoding might
619 // produce 3 characters.
620 result.append(input.substr(0, std::distance(input.begin(), pointer)));
621 if (static_cast<size_t>(input.end() - pointer) >= 48) {
622 percent_encode_suffix(&*pointer, input.data() + input.size(), character_set,
623 result);
624 } else {
625 for (; pointer != input.end(); pointer++) {
626 if (character_sets::bit_at(character_set, *pointer)) {
627 result.append(character_sets::hex + uint8_t(*pointer) * 4, 3);
628 } else {
629 result += *pointer;
630 }
631 }
632 }
633 return result;
634}
635
636template <bool append>
637bool percent_encode(const std::string_view input, const uint8_t character_set[],
638 std::string& out) {
639 ada_log("percent_encode ", input, " to output string while ",
640 append ? "appending" : "overwriting");
641 auto pointer = std::ranges::find_if(input, [character_set](const char c) {
642 return character_sets::bit_at(character_set, c);
643 });
644 ada_log("percent_encode done checking, moved to ",
645 std::distance(input.begin(), pointer));
646
647 // Optimization: Don't iterate if percent encode is not required
648 if (pointer == input.end()) {
649 ada_log("percent_encode encoding not needed.");
650 return false;
651 }
652 if constexpr (!append) {
653 out.clear();
654 }
655 ada_log("percent_encode appending ", std::distance(input.begin(), pointer),
656 " bytes");
657 // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
658 out.append(input.data(), std::distance(input.begin(), pointer));
659 ada_log("percent_encode processing ", std::distance(pointer, input.end()),
660 " bytes");
661 for (; pointer != input.end(); pointer++) {
662 if (character_sets::bit_at(character_set, *pointer)) {
663 out.append(character_sets::hex + uint8_t(*pointer) * 4, 3);
664 } else {
665 out += *pointer;
666 }
667 }
668 return true;
669}
670
671template bool percent_encode<true>(std::string_view input,
672 const uint8_t character_set[],
673 std::string& out);
674template bool percent_encode<false>(std::string_view input,
675 const uint8_t character_set[],
676 std::string& out);
677
678bool to_ascii(std::optional<std::string>& out, const std::string_view plain,
679 size_t first_percent) {
680 std::string percent_decoded_buffer;
681 std::string_view input = plain;
682 if (first_percent != std::string_view::npos) {
683 percent_decoded_buffer = unicode::percent_decode(plain, first_percent);
684 input = percent_decoded_buffer;
685 }
686 // input is a non-empty UTF-8 string, must be percent decoded
687 std::string idna_ascii = ada::idna::to_ascii(input);
688 if (idna_ascii.empty() || contains_forbidden_domain_code_point(
689 idna_ascii.data(), idna_ascii.size())) {
690 return false;
691 }
692 out = std::move(idna_ascii);
693 return true;
694}
695
696std::string percent_encode(const std::string_view input,
697 const uint8_t character_set[], size_t index) {
698 std::string out;
699 // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
700 out.append(input.data(), index);
701 auto pointer = input.begin() + index;
702 if (static_cast<size_t>(input.end() - pointer) >= 48) {
703 percent_encode_suffix(&*pointer, input.data() + input.size(), character_set,
704 out);
705 } else {
706 for (; pointer != input.end(); pointer++) {
707 if (character_sets::bit_at(character_set, *pointer)) {
708 out.append(character_sets::hex + uint8_t(*pointer) * 4, 3);
709 } else {
710 out += *pointer;
711 }
712 }
713 }
714 return out;
715}
716
717} // namespace ada::unicode
Definitions of the character sets used by unicode functions.
Declaration of the character sets used by unicode functions.
Cross-platform compiler macros and common definitions.
#define ADA_PUSH_DISABLE_ALL_WARNINGS
Definition common_defs.h:94
#define ADA_POP_DISABLE_WARNINGS
#define ada_really_inline
Definition common_defs.h:85
ada_really_inline constexpr bool bit_at(const uint8_t a[], const uint8_t i)
constexpr char hex[1024]
std::string to_ascii(std::string_view ut8_string)
Includes the declarations for unicode operations.
static constexpr std::array< uint8_t, 256 > is_forbidden_domain_code_point_table
Definition unicode.cpp:278
static constexpr std::array< uint8_t, 256 > is_forbidden_domain_code_point_table_or_upper
Definition unicode.cpp:318
std::string form_urlencoded_decode(std::string_view input)
Definition unicode.cpp:538
static constexpr char hex_to_binary_table[]
Definition unicode.cpp:461
template bool percent_encode< true >(std::string_view input, const uint8_t character_set[], std::string &out)
constexpr uint64_t broadcast(uint8_t v) noexcept
Definition unicode.cpp:41
constexpr std::string_view table_is_double_dot_path_segment[]
Definition unicode.cpp:400
static constexpr std::array< uint8_t, 256 > unhex_table
Definition unicode.cpp:523
constexpr bool is_tabs_or_newline(char c) noexcept
Definition unicode.cpp:37
static constexpr std::array< uint8_t, 256 > is_forbidden_host_code_point_table
Definition unicode.cpp:263
static constexpr std::array< bool, 256 > is_alnum_plus_table
Definition unicode.cpp:359
void percent_encode_suffix(const char *p, const char *end, const uint8_t character_set[], std::string &out)
template bool percent_encode< false >(std::string_view input, const uint8_t character_set[], std::string &out)
tl::expected< result_type, ada::errors > result
Definitions for all unicode specific functions.