Boost.Locale: boost/locale/utf.hpp Source File

| Boost.Locale

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files
- Examples

- File List

utf.hpp
1 //
2 // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh)
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See
5 // accompanying file LICENSE_1_0.txt or copy at
6 // http://www.boost.org/LICENSE_1_0.txt)
7 //
8 #ifndef BOOST_LOCALE_UTF_HPP_INCLUDED
9 #define BOOST_LOCALE_UTF_HPP_INCLUDED
10
11 #include <boost/cstdint.hpp>
12
13 namespace boost {
14 namespace locale {
20 namespace utf {
22 #ifdef __GNUC__
23 # define BOOST_LOCALE_LIKELY(x) __builtin_expect((x),1)
24 # define BOOST_LOCALE_UNLIKELY(x) __builtin_expect((x),0)
25 #else
26 # define BOOST_LOCALE_LIKELY(x) (x)
27 # define BOOST_LOCALE_UNLIKELY(x) (x)
28 #endif
29
34 typedef uint32_t code_point;
35
39 static const code_point illegal = 0xFFFFFFFFu;
40
44 static const code_point incomplete = 0xFFFFFFFEu;
45
49 inline bool is_valid_codepoint(code_point v)
50 {
51 if(v>0x10FFFF)
52 return false;
53 if(0xD800 <=v && v<= 0xDFFF) // surragates
54 return false;
55 return true;
56 }
57
58 #ifdef BOOST_LOCALE_DOXYGEN
59 template<typename CharType,int size=sizeof(CharType)>
63 struct utf_traits {
67 typedef CharType char_type;
82 template<typename Iterator>
83 static code_point decode(Iterator &p,Iterator e);
84
92 static const int max_width;
99 static int width(code_point value);
100
106 static int trail_length(char_type c);
110 static bool is_trail(char_type c);
114 static bool is_lead(char_type c);
115
126 template<typename Iterator>
127 static Iterator encode(code_point value,Iterator out);
133 template<typename Iterator>
134 static code_point decode_valid(Iterator &p);
135 };
136
137 #else
138
139 template<typename CharType,int size=sizeof(CharType)>
140 struct utf_traits;
141
142 template<typename CharType>
143 struct utf_traits<CharType,1> {
144
145 typedef CharType char_type;
146
147 static int trail_length(char_type ci)
148 {
149 unsigned char c = ci;
150 if(c < 128)
151 return 0;
152 if(BOOST_LOCALE_UNLIKELY(c < 194))
153 return -1;
154 if(c < 224)
155 return 1;
156 if(c < 240)
157 return 2;
158 if(BOOST_LOCALE_LIKELY(c <=244))
159 return 3;
160 return -1;
161 }
162
163 static const int max_width = 4;
164
165 static int width(code_point value)
166 {
167 if(value <=0x7F) {
168 return 1;
169 }
170 else if(value <=0x7FF) {
171 return 2;
172 }
173 else if(BOOST_LOCALE_LIKELY(value <=0xFFFF)) {
174 return 3;
175 }
176 else {
177 return 4;
178 }
179 }
180
181 static bool is_trail(char_type ci)
182 {
183 unsigned char c=ci;
184 return (c & 0xC0)==0x80;
185 }
186
187 static bool is_lead(char_type ci)
188 {
189 return !is_trail(ci);
190 }
191
192 template<typename Iterator>
193 static code_point decode(Iterator &p,Iterator e)
194 {
195 if(BOOST_LOCALE_UNLIKELY(p==e))
196 return incomplete;
197
198 unsigned char lead = *p++;
199
200 // First byte is fully validated here
201 int trail_size = trail_length(lead);
202
203 if(BOOST_LOCALE_UNLIKELY(trail_size < 0))
204 return illegal;
205
206 //
207 // Ok as only ASCII may be of size = 0
208 // also optimize for ASCII text
209 //
210 if(trail_size == 0)
211 return lead;
212
213 code_point c = lead & ((1<<(6-trail_size))-1);
214
215 // Read the rest
216 unsigned char tmp;
217 switch(trail_size) {
218 case 3:
219 if(BOOST_LOCALE_UNLIKELY(p==e))
220 return incomplete;
221 tmp = *p++;
222 if (!is_trail(tmp))
223 return illegal;
224 c = (c << 6) | ( tmp & 0x3F);
225 case 2:
226 if(BOOST_LOCALE_UNLIKELY(p==e))
227 return incomplete;
228 tmp = *p++;
229 if (!is_trail(tmp))
230 return illegal;
231 c = (c << 6) | ( tmp & 0x3F);
232 case 1:
233 if(BOOST_LOCALE_UNLIKELY(p==e))
234 return incomplete;
235 tmp = *p++;
236 if (!is_trail(tmp))
237 return illegal;
238 c = (c << 6) | ( tmp & 0x3F);
239 }
240
241 // Check code point validity: no surrogates and
242 // valid range
243 if(BOOST_LOCALE_UNLIKELY(!is_valid_codepoint(c)))
244 return illegal;
245
246 // make sure it is the most compact representation
247 if(BOOST_LOCALE_UNLIKELY(width(c)!=trail_size + 1))
248 return illegal;
249
250 return c;
251
252 }
253
254 template<typename Iterator>
255 static code_point decode_valid(Iterator &p)
256 {
257 unsigned char lead = *p++;
258 if(lead < 192)
259 return lead;
260
261 int trail_size;
262
263 if(lead < 224)
264 trail_size = 1;
265 else if(BOOST_LOCALE_LIKELY(lead < 240)) // non-BMP rare
266 trail_size = 2;
267 else
268 trail_size = 3;
269
270 code_point c = lead & ((1<<(6-trail_size))-1);
271
272 switch(trail_size) {
273 case 3:
274 c = (c << 6) | ( static_cast<unsigned char>(*p++) & 0x3F);
275 case 2:
276 c = (c << 6) | ( static_cast<unsigned char>(*p++) & 0x3F);
277 case 1:
278 c = (c << 6) | ( static_cast<unsigned char>(*p++) & 0x3F);
279 }
280
281 return c;
282 }
283
284
285
286 template<typename Iterator>
287 static Iterator encode(code_point value,Iterator out)
288 {
289 if(value <= 0x7F) {
290 *out++ = static_cast<char_type>(value);
291 }
292 else if(value <= 0x7FF) {
293 *out++ = static_cast<char_type>((value >> 6) | 0xC0);
294 *out++ = static_cast<char_type>((value & 0x3F) | 0x80);
295 }
296 else if(BOOST_LOCALE_LIKELY(value <= 0xFFFF)) {
297 *out++ = static_cast<char_type>((value >> 12) | 0xE0);
298 *out++ = static_cast<char_type>(((value >> 6) & 0x3F) | 0x80);
299 *out++ = static_cast<char_type>((value & 0x3F) | 0x80);
300 }
301 else {
302 *out++ = static_cast<char_type>((value >> 18) | 0xF0);
303 *out++ = static_cast<char_type>(((value >> 12) & 0x3F) | 0x80);
304 *out++ = static_cast<char_type>(((value >> 6) & 0x3F) | 0x80);
305 *out++ = static_cast<char_type>((value & 0x3F) | 0x80);
306 }
307 return out;
308 }
309 }; // utf8
310
311 template<typename CharType>
312 struct utf_traits<CharType,2> {
313 typedef CharType char_type;
314
315 // See RFC 2781
316 static bool is_first_surrogate(uint16_t x)
317 {
318 return 0xD800 <=x && x<= 0xDBFF;
319 }
320 static bool is_second_surrogate(uint16_t x)
321 {
322 return 0xDC00 <=x && x<= 0xDFFF;
323 }
324 static code_point combine_surrogate(uint16_t w1,uint16_t w2)
325 {
326 return ((code_point(w1 & 0x3FF) << 10) | (w2 & 0x3FF)) + 0x10000;
327 }
328 static int trail_length(char_type c)
329 {
330 if(is_first_surrogate(c))
331 return 1;
332 if(is_second_surrogate(c))
333 return -1;
334 return 0;
335 }
339 static bool is_trail(char_type c)
340 {
341 return is_second_surrogate(c);
342 }
346 static bool is_lead(char_type c)
347 {
348 return !is_second_surrogate(c);
349 }
350
351 template<typename It>
352 static code_point decode(It &current,It last)
353 {
354 if(BOOST_LOCALE_UNLIKELY(current == last))
355 return incomplete;
356 uint16_t w1=*current++;
357 if(BOOST_LOCALE_LIKELY(w1 < 0xD800 || 0xDFFF < w1)) {
358 return w1;
359 }
360 if(w1 > 0xDBFF)
361 return illegal;
362 if(current==last)
363 return incomplete;
364 uint16_t w2=*current++;
365 if(w2 < 0xDC00 || 0xDFFF < w2)
366 return illegal;
367 return combine_surrogate(w1,w2);
368 }
369 template<typename It>
370 static code_point decode_valid(It &current)
371 {
372 uint16_t w1=*current++;
373 if(BOOST_LOCALE_LIKELY(w1 < 0xD800 || 0xDFFF < w1)) {
374 return w1;
375 }
376 uint16_t w2=*current++;
377 return combine_surrogate(w1,w2);
378 }
379
380 static const int max_width = 2;
381 static int width(code_point u)
382 {
383 return u>=0x10000 ? 2 : 1;
384 }
385 template<typename It>
386 static It encode(code_point u,It out)
387 {
388 if(BOOST_LOCALE_LIKELY(u<=0xFFFF)) {
389 *out++ = static_cast<char_type>(u);
390 }
391 else {
392 u -= 0x10000;
393 *out++ = static_cast<char_type>(0xD800 | (u>>10));
394 *out++ = static_cast<char_type>(0xDC00 | (u & 0x3FF));
395 }
396 return out;
397 }
398 }; // utf16;
399
400
401 template<typename CharType>
402 struct utf_traits<CharType,4> {
403 typedef CharType char_type;
404 static int trail_length(char_type c)
405 {
406 if(is_valid_codepoint(c))
407 return 0;
408 return -1;
409 }
410 static bool is_trail(char_type /*c*/)
411 {
412 return false;
413 }
414 static bool is_lead(char_type /*c*/)
415 {
416 return true;
417 }
418
419 template<typename It>
420 static code_point decode_valid(It &current)
421 {
422 return *current++;
423 }
424
425 template<typename It>
426 static code_point decode(It &current,It last)
427 {
428 if(BOOST_LOCALE_UNLIKELY(current == last))
429 return boost::locale::utf::incomplete;
430 code_point c=*current++;
431 if(BOOST_LOCALE_UNLIKELY(!is_valid_codepoint(c)))
432 return boost::locale::utf::illegal;
433 return c;
434 }
435 static const int max_width = 1;
436 static int width(code_point /*u*/)
437 {
438 return 1;
439 }
440 template<typename It>
441 static It encode(code_point u,It out)
442 {
443 *out++ = static_cast<char_type>(u);
444 return out;
445 }
446
447 }; // utf32
448
449 #endif
450
451
452 } // utf
453 } // locale
454 } // boost
455
456
457 #endif
458
459 // vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
460
boost::locale::utf::utf_traits::decode
static code_point decode(Iterator &p, Iterator e)
boost::locale::utf::is_valid_codepoint
bool is_valid_codepoint(code_point v)
the function checks if v is a valid code point
Definition: utf.hpp:49
boost::locale::utf::utf_traits::encode
static Iterator encode(code_point value, Iterator out)
boost::locale::utf::incomplete
static const code_point incomplete
Special constant that defines incomplete code point.
Definition: utf.hpp:44
boost::locale::utf::code_point
uint32_t code_point
The integral type that can hold a Unicode code point.
Definition: utf.hpp:34
boost::locale::utf::illegal
static const code_point illegal
Special constant that defines illegal code point.
Definition: utf.hpp:39
boost::locale::utf::utf_traits
UTF Traits class - functions to convert UTF sequences to and from Unicode code points.
Definition: utf.hpp:63
boost::locale::utf::utf_traits::width
static int width(code_point value)
boost::locale::utf::utf_traits::char_type
CharType char_type
Definition: utf.hpp:67
boost::locale::utf::utf_traits::trail_length
static int trail_length(char_type c)
boost::locale::utf::utf_traits::is_lead
static bool is_lead(char_type c)
boost::locale::utf::utf_traits::decode_valid
static code_point decode_valid(Iterator &p)
boost::locale::utf::utf_traits::is_trail
static bool is_trail(char_type c)
boost::locale::utf::utf_traits::max_width
static const int max_width
Definition: utf.hpp:92
- © Copyright 2009-2012 Artyom Beilis, Distributed under the Boost Software License, Version 1.0.