2 * Copyright (C) 2001 Edmund Grimley Evans <edmundo@rano.org>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License along
15 * with this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 * See the corresponding header file for a description of the functions
21 * that this file provides.
23 * This was first written for Ogg Vorbis but could be of general use.
25 * The only deliberate assumption about data sizes is that a short has
26 * at least 16 bits, but this code has only been tested on systems with
27 * 8-bit char, 16-bit short and 32-bit int.
34 #if !defined _WIN32 && !defined HAVE_ICONV /* should be && defined USE_CHARSET_CONVERT */
38 #include "share/alloc.h"
44 * This is like the standard strcasecmp, but it does not depend
45 * on the locale. Locale-dependent functions can be dangerous:
46 * we once had a bug involving strcasecmp("iso", "ISO") in a
49 * (I'm not really sure what the official standard says
50 * about the sign of strcasecmp("Z", "["), but usually
51 * we're only interested in whether it's zero.)
54 static int ascii_strcasecmp(const char *s1, const char *s2)
64 if ('a' <= c1 && c1 <= 'z')
67 if ('a' <= c2 && c2 <= 'z')
72 return (unsigned char)*s1 - (unsigned char)*s2;
76 * UTF-8 equivalents of the C library's wctomb() and mbtowc().
79 int utf8_mbtowc(int *pwc, const char *s, size_t n)
96 if (n >= 2 && (s[1] & 0xc0) == 0x80) {
98 *pwc = ((c & 0x1f) << 6) | (s[1] & 0x3f);
117 wc = *s++ & ((1 << (7 - k)) - 1);
118 for (i = 1; i < k; i++) {
119 if ((*s & 0xc0) != 0x80)
121 wc = (wc << 6) | (*s++ & 0x3f);
123 if (wc < (1 << (5 * k - 4)))
130 int utf8_wctomb(char *s, int wc1)
132 unsigned int wc = wc1;
136 if (wc < (1u << 7)) {
140 else if (wc < (1u << 11)) {
141 *s++ = 0xc0 | (wc >> 6);
142 *s++ = 0x80 | (wc & 0x3f);
145 else if (wc < (1u << 16)) {
146 *s++ = 0xe0 | (wc >> 12);
147 *s++ = 0x80 | ((wc >> 6) & 0x3f);
148 *s++ = 0x80 | (wc & 0x3f);
151 else if (wc < (1u << 21)) {
152 *s++ = 0xf0 | (wc >> 18);
153 *s++ = 0x80 | ((wc >> 12) & 0x3f);
154 *s++ = 0x80 | ((wc >> 6) & 0x3f);
155 *s++ = 0x80 | (wc & 0x3f);
158 else if (wc < (1u << 26)) {
159 *s++ = 0xf8 | (wc >> 24);
160 *s++ = 0x80 | ((wc >> 18) & 0x3f);
161 *s++ = 0x80 | ((wc >> 12) & 0x3f);
162 *s++ = 0x80 | ((wc >> 6) & 0x3f);
163 *s++ = 0x80 | (wc & 0x3f);
166 else if (wc < (1u << 31)) {
167 *s++ = 0xfc | (wc >> 30);
168 *s++ = 0x80 | ((wc >> 24) & 0x3f);
169 *s++ = 0x80 | ((wc >> 18) & 0x3f);
170 *s++ = 0x80 | ((wc >> 12) & 0x3f);
171 *s++ = 0x80 | ((wc >> 6) & 0x3f);
172 *s++ = 0x80 | (wc & 0x3f);
180 * The charset "object" and methods.
185 int (*mbtowc)(void *table, int *pwc, const char *s, size_t n);
186 int (*wctomb)(void *table, char *s, int wc);
190 int charset_mbtowc(struct charset *charset, int *pwc, const char *s, size_t n)
192 return (*charset->mbtowc)(charset->map, pwc, s, n);
195 int charset_wctomb(struct charset *charset, char *s, int wc)
197 return (*charset->wctomb)(charset->map, s, wc);
200 int charset_max(struct charset *charset)
206 * Implementation of UTF-8.
209 static int mbtowc_utf8(void *map, int *pwc, const char *s, size_t n)
212 return utf8_mbtowc(pwc, s, n);
215 static int wctomb_utf8(void *map, char *s, int wc)
218 return utf8_wctomb(s, wc);
222 * Implementation of US-ASCII.
223 * Probably on most architectures this compiles to less than 256 bytes
224 * of code, so we can save space by not having a table for this one.
227 static int mbtowc_ascii(void *map, int *pwc, const char *s, size_t n)
234 wc = (unsigned char)*s;
242 static int wctomb_ascii(void *map, char *s, int wc)
254 * Implementation of ISO-8859-1.
255 * Probably on most architectures this compiles to less than 256 bytes
256 * of code, so we can save space by not having a table for this one.
259 static int mbtowc_iso1(void *map, int *pwc, const char *s, size_t n)
266 wc = (unsigned char)*s;
274 static int wctomb_iso1(void *map, char *s, int wc)
286 * Implementation of any 8-bit charset.
290 const unsigned short *from;
291 struct inverse_map *to;
294 static int mbtowc_8bit(void *map1, int *pwc, const char *s, size_t n)
296 struct map *map = map1;
301 wc = map->from[(unsigned char)*s];
310 * For the inverse map we use a hash table, which has the advantages
311 * of small constant memory requirement and simple memory allocation,
312 * but the disadvantage of slow conversion in the worst case.
313 * If you need real-time performance while letting a potentially
314 * malicious user define their own map, then the method used in
315 * linux/drivers/char/consolemap.c would be more appropriate.
319 unsigned char first[256];
320 unsigned char next[256];
324 * The simple hash is good enough for this application.
325 * Use the alternative trivial hashes for testing.
327 #define HASH(i) ((i) & 0xff)
328 /* #define HASH(i) 0 */
329 /* #define HASH(i) 99 */
331 static struct inverse_map *make_inverse_map(const unsigned short *from)
333 struct inverse_map *to;
337 to = malloc(sizeof(struct inverse_map));
340 for (i = 0; i < 256; i++)
341 to->first[i] = to->next[i] = used[i] = 0;
342 for (i = 255; i >= 0; i--)
343 if (from[i] != 0xffff) {
345 to->next[i] = to->first[k];
350 /* Point the empty buckets at an empty list. */
351 for (i = 0; i < 256; i++)
355 for (j = 0; j < 256; j++)
362 static int wctomb_8bit(void *map1, char *s, int wc1)
364 struct map *map = map1;
365 unsigned short wc = wc1;
374 if (1) /* Change 1 to 0 to test the case where malloc fails. */
376 map->to = make_inverse_map(map->from);
379 /* Use the inverse map. */
380 i = map->to->first[HASH(wc)];
382 if (map->from[i] == wc) {
386 if (!(i = map->to->next[i]))
391 /* We don't have an inverse map, so do a linear search. */
392 for (i = 0; i < 256; i++)
393 if (map->from[i] == wc) {
403 * The "constructor" charset_find().
406 struct charset charset_utf8 = {
413 struct charset charset_iso1 = {
420 struct charset charset_ascii = {
427 struct charset *charset_find(const char *code)
431 /* Find good (MIME) name. */
432 for (i = 0; names[i].bad; i++)
433 if (!ascii_strcasecmp(code, names[i].bad)) {
434 code = names[i].good;
438 /* Recognise some charsets for which we avoid using a table. */
439 if (!ascii_strcasecmp(code, "UTF-8"))
440 return &charset_utf8;
441 if (!ascii_strcasecmp(code, "US-ASCII"))
442 return &charset_ascii;
443 if (!ascii_strcasecmp(code, "ISO-8859-1"))
444 return &charset_iso1;
446 /* Look for a mapping for a simple 8-bit encoding. */
447 for (i = 0; maps[i].name; i++)
448 if (!ascii_strcasecmp(code, maps[i].name)) {
449 if (!maps[i].charset) {
450 maps[i].charset = malloc(sizeof(struct charset));
451 if (maps[i].charset) {
452 struct map *map = malloc(sizeof(struct map));
454 free(maps[i].charset);
458 maps[i].charset->max = 1;
459 maps[i].charset->mbtowc = &mbtowc_8bit;
460 maps[i].charset->wctomb = &wctomb_8bit;
461 maps[i].charset->map = map;
462 map->from = maps[i].map;
463 map->to = 0; /* inverse mapping is created when required */
467 return maps[i].charset;
474 * Function to convert a buffer from one encoding to another.
475 * Invalid bytes are replaced by '#', and characters that are
476 * not available in the target encoding are replaced by '?'.
477 * Each of TO and TOLEN may be zero, if the result is not needed.
478 * The output buffer is null-terminated, so it is all right to
479 * use charset_convert(fromcode, tocode, s, strlen(s), &t, 0).
482 int charset_convert(const char *fromcode, const char *tocode,
483 const char *from, size_t fromlen,
484 char **to, size_t *tolen)
487 struct charset *charset1, *charset2;
488 char *tobuf, *p, *newbuf;
491 charset1 = charset_find(fromcode);
492 charset2 = charset_find(tocode);
493 if (!charset1 || !charset2 )
496 tobuf = safe_malloc_mul2add_(fromlen, /*times*/charset2->max, /*+*/1);
500 for (p = tobuf; fromlen; from += i, fromlen -= i, p += j) {
501 i = charset_mbtowc(charset1, &wc, from, fromlen);
509 j = charset_wctomb(charset2, p, wc);
513 j = charset_wctomb(charset2, p, '?');
523 newbuf = realloc(tobuf, p - tobuf);
524 *to = newbuf ? newbuf : tobuf;
532 #endif /* USE_CHARSET_ICONV */