9size_t utf8_to_utf32(
const char* buf,
size_t len,
char32_t* utf32_output) {
10 const uint8_t* data =
reinterpret_cast<const uint8_t*
>(buf);
12 const char32_t* start{utf32_output};
15 if (pos + 16 <= len) {
18 std::memcpy(&v1, data + pos,
sizeof(uint64_t));
20 std::memcpy(&v2, data + pos +
sizeof(uint64_t),
sizeof(uint64_t));
22 if ((v & 0x8080808080808080) == 0) {
23 size_t final_pos = pos + 16;
24 while (pos < final_pos) {
25 *utf32_output++ = char32_t(buf[pos]);
31 uint8_t leading_byte = data[pos];
32 if (leading_byte < 0b10000000) {
34 *utf32_output++ = char32_t(leading_byte);
36 }
else if ((leading_byte & 0b11100000) == 0b11000000) {
41 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
46 (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
47 if (code_point < 0x80 || 0x7ff < code_point) {
50 *utf32_output++ = char32_t(code_point);
52 }
else if ((leading_byte & 0b11110000) == 0b11100000) {
58 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
61 if ((data[pos + 2] & 0b11000000) != 0b10000000) {
65 uint32_t code_point = (leading_byte & 0b00001111) << 12 |
66 (data[pos + 1] & 0b00111111) << 6 |
67 (data[pos + 2] & 0b00111111);
68 if (code_point < 0x800 || 0xffff < code_point ||
69 (0xd7ff < code_point && code_point < 0xe000)) {
72 *utf32_output++ = char32_t(code_point);
74 }
else if ((leading_byte & 0b11111000) == 0b11110000) {
79 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
82 if ((data[pos + 2] & 0b11000000) != 0b10000000) {
85 if ((data[pos + 3] & 0b11000000) != 0b10000000) {
90 uint32_t code_point = (leading_byte & 0b00000111) << 18 |
91 (data[pos + 1] & 0b00111111) << 12 |
92 (data[pos + 2] & 0b00111111) << 6 |
93 (data[pos + 3] & 0b00111111);
94 if (code_point <= 0xffff || 0x10ffff < code_point) {
97 *utf32_output++ = char32_t(code_point);
103 return utf32_output - start;
108 const uint32_t* p =
reinterpret_cast<const uint32_t*
>(buf);
110 for (
size_t i = 0; i != len; ++i) {
112 counter +=
static_cast<size_t>(p[i] > 0x7F);
113 counter +=
static_cast<size_t>(p[i] > 0x7FF);
114 counter +=
static_cast<size_t>(p[i] > 0xFFFF);
120 const int8_t* p =
reinterpret_cast<const int8_t*
>(buf);
121 return std::count_if(p, std::next(p, len), [](int8_t c) {
128size_t utf32_to_utf8(
const char32_t* buf,
size_t len,
char* utf8_output) {
129 const uint32_t* data =
reinterpret_cast<const uint32_t*
>(buf);
131 const char* start{utf8_output};
134 if (pos + 2 <= len) {
137 std::memcpy(&v, data + pos,
sizeof(uint64_t));
138 if ((v & 0xFFFFFF80FFFFFF80) == 0) {
139 *utf8_output++ = char(buf[pos]);
140 *utf8_output++ = char(buf[pos + 1]);
145 uint32_t word = data[pos];
146 if ((word & 0xFFFFFF80) == 0) {
148 *utf8_output++ = char(word);
150 }
else if ((word & 0xFFFFF800) == 0) {
153 *utf8_output++ = char((word >> 6) | 0b11000000);
154 *utf8_output++ = char((word & 0b111111) | 0b10000000);
156 }
else if ((word & 0xFFFF0000) == 0) {
159 if (word >= 0xD800 && word <= 0xDFFF) {
162 *utf8_output++ = char((word >> 12) | 0b11100000);
163 *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
164 *utf8_output++ = char((word & 0b111111) | 0b10000000);
170 if (word > 0x10FFFF) {
173 *utf8_output++ = char((word >> 18) | 0b11110000);
174 *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000);
175 *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
176 *utf8_output++ = char((word & 0b111111) | 0b10000000);
180 return utf8_output - start;
size_t utf32_length_from_utf8(const char *buf, size_t len)
size_t utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_output)
size_t utf8_length_from_utf32(const char32_t *buf, size_t len)
size_t utf8_to_utf32(const char *buf, size_t len, char32_t *utf32_output)