1 /* Copyright (C) 2010-2020 The RetroArch team
3 * ---------------------------------------------------------------------------------------
4 * The following license statement only applies to this file (encoding_utf.c).
5 * ---------------------------------------------------------------------------------------
7 * Permission is hereby granted, free of charge,
8 * to any person obtaining a copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation the rights to
10 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
11 * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
16 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
19 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 #include <compat/strl.h>
30 #include <retro_inline.h>
32 #include <encodings/utf.h>
34 #if defined(_WIN32) && !defined(_XBOX)
40 #define UTF8_WALKBYTE(string) (*((*(string))++))
42 static unsigned leading_ones(uint8_t c)
57 * Simple implementation. Assumes the sequence is
58 * properly synchronized and terminated.
60 size_t utf8_conv_utf32(uint32_t *out, size_t out_chars,
61 const char *in, size_t in_size)
65 while (in_size && out_chars)
67 unsigned extra, shift;
69 uint8_t first = *in++;
70 unsigned ones = leading_ones(first);
72 if (ones > 6 || ones == 1) /* Invalid or desync. */
75 extra = ones ? ones - 1 : ones;
76 if (1 + extra > in_size) /* Overflow. */
79 shift = (extra - 1) * 6;
80 c = (first & ((1 << (7 - ones)) - 1)) << (6 * extra);
82 for (i = 0; i < extra; i++, in++, shift -= 6)
83 c |= (*in & 0x3f) << shift;
99 bool utf16_conv_utf8(uint8_t *out, size_t *out_chars,
100 const uint16_t *in, size_t in_size)
105 uint8_t utf8_limits[5] = { 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
112 if (in_pos == in_size)
114 *out_chars = out_pos;
117 value = in[in_pos++];
121 out[out_pos] = (char)value;
126 if (value >= 0xD800 && value < 0xE000)
130 if (value >= 0xDC00 || in_pos == in_size)
133 if (c2 < 0xDC00 || c2 >= 0xE000)
135 value = (((value - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000;
138 for (num_adds = 1; num_adds < 5; num_adds++)
139 if (value < (((uint32_t)1) << (num_adds * 5 + 6)))
142 out[out_pos] = (char)(utf8_limits[num_adds - 1]
143 + (value >> (6 * num_adds)));
149 out[out_pos] = (char)(0x80
150 + ((value >> (6 * num_adds)) & 0x3F));
152 }while (num_adds != 0);
155 *out_chars = out_pos;
162 * Acts mostly like strlcpy.
164 * Copies the given number of UTF-8 characters,
165 * but at most @d_len bytes.
167 * Always NULL terminates. Does not copy half a character.
168 * @s is assumed valid UTF-8.
169 * Use only if @chars is considerably less than @d_len.
171 * @return Number of bytes.
173 size_t utf8cpy(char *d, size_t d_len, const char *s, size_t chars)
175 const uint8_t *sb = (const uint8_t*)s;
176 const uint8_t *sb_org = sb;
181 while (*sb && chars-- > 0)
184 while ((*sb & 0xC0) == 0x80)
188 if ((size_t)(sb - sb_org) > d_len-1 /* NUL */)
190 sb = sb_org + d_len-1;
191 while ((*sb & 0xC0) == 0x80)
195 memcpy(d, sb_org, sb-sb_org);
206 const char *utf8skip(const char *str, size_t chars)
208 const uint8_t *strb = (const uint8_t*)str;
216 while ((*strb & 0xC0)==0x80)
221 return (const char*)strb;
229 size_t utf8len(const char *string)
238 if ((*string & 0xC0) != 0x80)
248 * Does not validate the input.
252 * @return Returns garbage if it's not UTF-8.
254 uint32_t utf8_walk(const char **string)
256 uint8_t first = UTF8_WALKBYTE(string);
262 ret = (ret << 6) | (UTF8_WALKBYTE(string) & 0x3F);
265 ret = (ret << 6) | (UTF8_WALKBYTE(string) & 0x3F);
268 ret = (ret << 6) | (UTF8_WALKBYTE(string) & 0x3F);
269 return ret | (first & 7) << 18;
271 return ret | (first & 15) << 12;
274 return ret | (first & 31) << 6;
277 static bool utf16_to_char(uint8_t **utf_data,
278 size_t *dest_len, const uint16_t *in)
281 while (in[len] != '\0')
283 utf16_conv_utf8(NULL, dest_len, in, len);
285 if ((*utf_data = (uint8_t*)malloc(*dest_len)) != 0)
286 return utf16_conv_utf8(*utf_data, dest_len, in, len);
291 * utf16_to_char_string:
293 bool utf16_to_char_string(const uint16_t *in, char *s, size_t len)
296 uint8_t *utf16_data = NULL;
297 bool ret = utf16_to_char(&utf16_data, &dest_len, in);
301 utf16_data[dest_len] = 0;
302 strlcpy(s, (const char*)utf16_data, len);
311 #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
313 * mb_to_mb_string_alloc:
315 * @return Returned pointer MUST be freed by the caller if non-NULL.
317 static char *mb_to_mb_string_alloc(const char *str,
318 enum CodePage cp_in, enum CodePage cp_out)
320 wchar_t *path_buf_wide = NULL;
321 int path_buf_wide_len = MultiByteToWideChar(cp_in, 0, str, -1, NULL, 0);
323 /* Windows 95 will return 0 from these functions with
324 * a UTF8 codepage set without MSLU.
326 * From an unknown MSDN version (others omit this info):
327 * - CP_UTF8 Windows 98/Me, Windows NT 4.0 and later:
328 * Translate using UTF-8. When this is set, dwFlags must be zero.
329 * - Windows 95: Under the Microsoft Layer for Unicode,
330 * MultiByteToWideChar also supports CP_UTF7 and CP_UTF8.
333 if (!path_buf_wide_len)
336 if ((path_buf_wide = (wchar_t*)
337 calloc(path_buf_wide_len + sizeof(wchar_t), sizeof(wchar_t))))
339 MultiByteToWideChar(cp_in, 0,
340 str, -1, path_buf_wide, path_buf_wide_len);
344 int path_buf_len = WideCharToMultiByte(cp_out, 0,
345 path_buf_wide, -1, NULL, 0, NULL, NULL);
349 char *path_buf = (char*)
350 calloc(path_buf_len + sizeof(char), sizeof(char));
354 WideCharToMultiByte(cp_out, 0,
355 path_buf_wide, -1, path_buf,
356 path_buf_len, NULL, NULL);
382 * utf8_to_local_string_alloc:
384 * @return Returned pointer MUST be freed by the caller if non-NULL.
386 char* utf8_to_local_string_alloc(const char *str)
389 #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
390 return mb_to_mb_string_alloc(str, CODEPAGE_UTF8, CODEPAGE_LOCAL);
392 return strdup(str); /* Assume string needs no modification if not on Windows */
398 * local_to_utf8_string_alloc:
400 * @return Returned pointer MUST be freed by the caller if non-NULL.
402 char *local_to_utf8_string_alloc(const char *str)
405 #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
406 return mb_to_mb_string_alloc(str, CODEPAGE_LOCAL, CODEPAGE_UTF8);
408 return strdup(str); /* Assume string needs no modification if not on Windows */
414 * utf8_to_utf16_string_alloc:
416 * @return Returned pointer MUST be freed by the caller if non-NULL.
418 wchar_t* utf8_to_utf16_string_alloc(const char *str)
431 if ((len = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0)))
433 if (!(buf = (wchar_t*)calloc(len, sizeof(wchar_t))))
436 if ((MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, len)) < 0)
444 /* Fallback to ANSI codepage instead */
445 if ((len = MultiByteToWideChar(CP_ACP, 0, str, -1, NULL, 0)))
447 if (!(buf = (wchar_t*)calloc(len, sizeof(wchar_t))))
450 if ((MultiByteToWideChar(CP_ACP, 0, str, -1, buf, len)) < 0)
458 /* NOTE: For now, assume non-Windows platforms' locale is already UTF-8. */
459 if ((len = mbstowcs(NULL, str, 0) + 1))
461 if (!(buf = (wchar_t*)calloc(len, sizeof(wchar_t))))
464 if ((mbstowcs(buf, str, len)) == (size_t)-1)
476 * utf16_to_utf8_string_alloc:
478 * @return Returned pointer MUST be freed by the caller if non-NULL.
480 char* utf16_to_utf8_string_alloc(const wchar_t *str)
494 UINT code_page = CP_UTF8;
496 /* fallback to ANSI codepage instead */
497 if (!(len = WideCharToMultiByte(code_page,
498 0, str, -1, NULL, 0, NULL, NULL)))
501 len = WideCharToMultiByte(code_page,
502 0, str, -1, NULL, 0, NULL, NULL);
505 if (!(buf = (char*)calloc(len, sizeof(char))))
508 if (WideCharToMultiByte(code_page,
509 0, str, -1, buf, len, NULL, NULL) < 0)
516 /* NOTE: For now, assume non-Windows platforms'
517 * locale is already UTF-8. */
518 if ((len = wcstombs(NULL, str, 0) + 1))
520 if (!(buf = (char*)calloc(len, sizeof(char))))
523 if (wcstombs(buf, str, len) == (size_t)-1)