1/*-
2 * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
3 * Copyright 2012 Garrett D'Amore <garrett@damore.org>  All rights reserved.
4 * Copyright 2015 John Marino <draco@marino.st>
5 *
6 * This source code is derived from the illumos localedef command, and
7 * provided under BSD-style license terms by Nexenta Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * The functions in this file convert from the standard multibyte forms
34 * to the wide character forms used internally by libc.  Unfortunately,
35 * this approach means that we need a method for each and every encoding.
36 */
37#include <sys/cdefs.h>
38#include <ctype.h>
39#include <stdlib.h>
40#include <wchar.h>
41#include <string.h>
42#include <sys/types.h>
43#include "localedef.h"
44
45static int towide_none(wchar_t *, const char *, unsigned);
46static int towide_utf8(wchar_t *, const char *, unsigned);
47static int towide_big5(wchar_t *, const char *, unsigned);
48static int towide_gbk(wchar_t *, const char *, unsigned);
49static int towide_gb2312(wchar_t *, const char *, unsigned);
50static int towide_gb18030(wchar_t *, const char *, unsigned);
51static int towide_mskanji(wchar_t *, const char *, unsigned);
52static int towide_euccn(wchar_t *, const char *, unsigned);
53static int towide_eucjp(wchar_t *, const char *, unsigned);
54static int towide_euckr(wchar_t *, const char *, unsigned);
55static int towide_euctw(wchar_t *, const char *, unsigned);
56
57static int tomb_none(char *, wchar_t);
58static int tomb_utf8(char *, wchar_t);
59static int tomb_mbs(char *, wchar_t);
60
61static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
62static int (*_tomb)(char *, wchar_t) = tomb_none;
63static char _encoding_buffer[20] = {'N','O','N','E'};
64static const char *_encoding = _encoding_buffer;
65static int _nbits = 7;
66
67/*
68 * Table of supported encodings.  We only bother to list the multibyte
69 * encodings here, because single byte locales are handed by "NONE".
70 */
71static struct {
72	const char *name;
73	/* the name that the underlying libc implemenation uses */
74	const char *cname;
75	/* the maximum number of bits required for priorities */
76	int nbits;
77	int (*towide)(wchar_t *, const char *, unsigned);
78	int (*tomb)(char *, wchar_t);
79} mb_encodings[] = {
80	/*
81	 * UTF8 values max out at 0x1fffff (although in theory there could
82	 * be later extensions, but it won't happen.)  This means we only need
83	 * 21 bits to be able to encode the entire range of priorities.
84	 */
85	{ "UTF-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
86	{ "UTF8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
87	{ "utf8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
88	{ "utf-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
89
90	{ "EUC-CN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
91	{ "eucCN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
92	/*
93	 * Because the 3-byte form of EUC-JP use the same leading byte,
94	 * only 17 bits required to provide unique priorities.  (The low
95	 * bit of that first byte is set.)  By setting this value low,
96	 * we can get by with only 3 bytes in the strxfrm expansion.
97	 */
98	{ "EUC-JP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
99	{ "eucJP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
100
101	{ "EUC-KR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
102	{ "eucKR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
103	/*
104	 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
105	 * high order byte is 0x8E.  However, with 4 byte encodings,
106	 * the third byte will be A0-B0.  So we only need to consider
107	 * the lower order 24 bits for collation.
108	 */
109	{ "EUC-TW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
110	{ "eucTW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
111
112	{ "MS_Kanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
113	{ "MSKanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
114	{ "PCK",	"MSKanji",	16, towide_mskanji, tomb_mbs },
115	{ "SJIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
116	{ "Shift_JIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
117
118	{ "BIG5",	"BIG5",		16, towide_big5, tomb_mbs },
119	{ "big5",	"BIG5",		16, towide_big5, tomb_mbs },
120	{ "Big5",	"BIG5",		16, towide_big5, tomb_mbs },
121
122	{ "GBK",	"GBK",		16, towide_gbk,	tomb_mbs },
123
124	/*
125	 * GB18030 can get away with just 31 bits.  This is because the
126	 * high order bit is always set for 4 byte values, and the
127	 * at least one of the other bits in that 4 byte value will
128	 * be non-zero.
129	 */
130	{ "GB18030",	"GB18030",	31, towide_gb18030, tomb_mbs },
131
132	/*
133	 * This should probably be an aliase for euc-cn, or vice versa.
134	 */
135	{ "GB2312",	"GB2312",	16, towide_gb2312, tomb_mbs },
136
137	{ NULL, NULL, 0, 0, 0 },
138};
139
140static char *
141show_mb(const char *mb)
142{
143	static char buf[64];
144
145	/* ASCII stuff we just print */
146	if (isascii(*mb) && isgraph(*mb)) {
147		buf[0] = *mb;
148		buf[1] = 0;
149		return (buf);
150	}
151	buf[0] = 0;
152	while (*mb != 0) {
153		char scr[8];
154		(void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
155		(void) strlcat(buf, scr, sizeof (buf));
156		mb++;
157	}
158	return (buf);
159}
160
161static char	*widemsg;
162
163void
164werr(const char *fmt, ...)
165{
166	char	*msg;
167
168	va_list	va;
169	va_start(va, fmt);
170	(void) vasprintf(&msg, fmt, va);
171	va_end(va);
172
173	free(widemsg);
174	widemsg = msg;
175}
176
177/*
178 * This is used for 8-bit encodings.
179 */
180int
181towide_none(wchar_t *c, const char *mb, unsigned n __unused)
182{
183	if (mb_cur_max != 1) {
184		werr("invalid or unsupported multibyte locale");
185		return (-1);
186	}
187	*c = (uint8_t)*mb;
188	return (1);
189}
190
191int
192tomb_none(char *mb, wchar_t wc)
193{
194	if (mb_cur_max != 1) {
195		werr("invalid or unsupported multibyte locale");
196		return (-1);
197	}
198	*(uint8_t *)mb = (wc & 0xff);
199	mb[1] = 0;
200	return (1);
201}
202
203/*
204 * UTF-8 stores wide characters in UTF-32 form.
205 */
206int
207towide_utf8(wchar_t *wc, const char *mb, unsigned n)
208{
209	wchar_t	c;
210	int	nb;
211	wchar_t	lv;	/* lowest legal value */
212	int	i;
213	const uint8_t *s = (const uint8_t *)mb;
214
215	c = *s;
216
217	if ((c & 0x80) == 0) {
218		/* 7-bit ASCII */
219		*wc = c;
220		return (1);
221	} else if ((c & 0xe0) == 0xc0) {
222		/* u80-u7ff - two bytes encoded */
223		nb = 2;
224		lv = 0x80;
225		c &= ~0xe0;
226	} else if ((c & 0xf0) == 0xe0) {
227		/* u800-uffff - three bytes encoded */
228		nb = 3;
229		lv = 0x800;
230		c &= ~0xf0;
231	} else if ((c & 0xf8) == 0xf0) {
232		/* u1000-u1fffff - four bytes encoded */
233		nb = 4;
234		lv = 0x1000;
235		c &= ~0xf8;
236	} else {
237		/* 5 and 6 byte encodings are not legal unicode */
238		werr("utf8 encoding too large (%s)", show_mb(mb));
239		return (-1);
240	}
241	if (nb > (int)n) {
242		werr("incomplete utf8 sequence (%s)", show_mb(mb));
243		return (-1);
244	}
245
246	for (i = 1; i < nb; i++) {
247		if (((s[i]) & 0xc0) != 0x80) {
248			werr("illegal utf8 byte (%x)", s[i]);
249			return (-1);
250		}
251		c <<= 6;
252		c |= (s[i] & 0x3f);
253	}
254
255	if (c < lv) {
256		werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
257		return (-1);
258	}
259	*wc = c;
260	return (nb);
261}
262
263int
264tomb_utf8(char *mb, wchar_t wc)
265{
266	uint8_t *s = (uint8_t *)mb;
267	uint8_t msk;
268	int cnt;
269	int i;
270
271	if (wc <= 0x7f) {
272		s[0] = wc & 0x7f;
273		s[1] = 0;
274		return (1);
275	}
276	if (wc <= 0x7ff) {
277		cnt = 2;
278		msk = 0xc0;
279	} else if (wc <= 0xffff) {
280		cnt = 3;
281		msk = 0xe0;
282	} else if (wc <= 0x1fffff) {
283		cnt = 4;
284		msk = 0xf0;
285	} else {
286		werr("illegal uf8 char (%x)", wc);
287		return (-1);
288	}
289	for (i = cnt - 1; i; i--) {
290		s[i] = (wc & 0x3f) | 0x80;
291		wc >>= 6;
292	}
293	s[0] = (msk) | wc;
294	s[cnt] = 0;
295	return (cnt);
296}
297
298/*
299 * Several encodings share a simplistic dual byte encoding.  In these
300 * forms, they all indicate that a two byte sequence is to be used if
301 * the first byte has its high bit set.  They all store this simple
302 * encoding as a 16-bit value, although a great many of the possible
303 * code points are not used in most character sets.  This gives a possible
304 * set of just over 32,000 valid code points.
305 *
306 * 0x00 - 0x7f		- 1 byte encoding
307 * 0x80 - 0x7fff	- illegal
308 * 0x8000 - 0xffff	- 2 byte encoding
309 */
310
311static int
312towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
313{
314	wchar_t	c;
315
316	c = *(const uint8_t *)mb;
317
318	if ((c & 0x80) == 0) {
319		/* 7-bit */
320		*wc = c;
321		return (1);
322	}
323	if (n < 2) {
324		werr("incomplete character sequence (%s)", show_mb(mb));
325		return (-1);
326	}
327
328	/* Store both bytes as a single 16-bit wide. */
329	c <<= 8;
330	c |= (uint8_t)(mb[1]);
331	*wc = c;
332	return (2);
333}
334
335/*
336 * Most multibyte locales just convert the wide character to the multibyte
337 * form by stripping leading null bytes, and writing the 32-bit quantity
338 * in big-endian order.
339 */
340int
341tomb_mbs(char *mb, wchar_t wc)
342{
343	uint8_t *s = (uint8_t *)mb;
344	int 	n = 0, c;
345
346	if ((wc & 0xff000000U) != 0) {
347		n = 4;
348	} else if ((wc & 0x00ff0000U) != 0) {
349		n = 3;
350	} else if ((wc & 0x0000ff00U) != 0) {
351		n = 2;
352	} else {
353		n = 1;
354	}
355	c = n;
356	while (n) {
357		n--;
358		s[n] = wc & 0xff;
359		wc >>= 8;
360	}
361	/* ensure null termination */
362	s[c] = 0;
363	return (c);
364}
365
366
367/*
368 * big5 is a simple dual byte character set.
369 */
370int
371towide_big5(wchar_t *wc, const char *mb, unsigned n)
372{
373	return (towide_dbcs(wc, mb, n));
374}
375
376/*
377 * GBK encodes wides in the same way that big5 does, the high order
378 * bit of the first byte indicates a double byte character.
379 */
380int
381towide_gbk(wchar_t *wc, const char *mb, unsigned n)
382{
383	return (towide_dbcs(wc, mb, n));
384}
385
386/*
387 * GB2312 is another DBCS.  Its cleaner than others in that the second
388 * byte does not encode ASCII, but it supports characters.
389 */
390int
391towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
392{
393	return (towide_dbcs(wc, mb, n));
394}
395
396/*
397 * GB18030.  This encodes as 8, 16, or 32-bits.
398 * 7-bit values are in 1 byte,  4 byte sequences are used when
399 * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
400 */
401int
402towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
403{
404	wchar_t	c;
405
406	c = *(const uint8_t *)mb;
407
408	if ((c & 0x80) == 0) {
409		/* 7-bit */
410		*wc = c;
411		return (1);
412	}
413	if (n < 2) {
414		werr("incomplete character sequence (%s)", show_mb(mb));
415		return (-1);
416	}
417
418	/* pull in the second byte */
419	c <<= 8;
420	c |= (uint8_t)(mb[1]);
421
422	if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
423		if (n < 4) {
424			werr("incomplete 4-byte character sequence (%s)",
425			    show_mb(mb));
426			return (-1);
427		}
428		c <<= 8;
429		c |= (uint8_t)(mb[2]);
430		c <<= 8;
431		c |= (uint8_t)(mb[3]);
432		*wc = c;
433		return (4);
434	}
435
436	*wc = c;
437	return (2);
438}
439
440/*
441 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
442 * also has a range of single byte characters above 0x80.  (0xa1-0xdf).
443 */
444int
445towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
446{
447	wchar_t	c;
448
449	c = *(const uint8_t *)mb;
450
451	if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
452		/* 7-bit */
453		*wc = c;
454		return (1);
455	}
456
457	if (n < 2) {
458		werr("incomplete character sequence (%s)", show_mb(mb));
459		return (-1);
460	}
461
462	/* Store both bytes as a single 16-bit wide. */
463	c <<= 8;
464	c |= (uint8_t)(mb[1]);
465	*wc = c;
466	return (2);
467}
468
469/*
470 * EUC forms.  EUC encodings are "variable".  FreeBSD carries some additional
471 * variable data to encode these, but we're going to treat each as independent
472 * instead.  Its the only way we can sensibly move forward.
473 *
474 * Note that the way in which the different EUC forms vary is how wide
475 * CS2 and CS3 are and what the first byte of them is.
476 */
477static int
478towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
479    uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
480{
481	int i;
482	int width = 2;
483	wchar_t	c;
484
485	c = *(const uint8_t *)mb;
486
487	/*
488	 * All variations of EUC encode 7-bit ASCII as one byte, and use
489	 * additional bytes for more than that.
490	 */
491	if ((c & 0x80) == 0) {
492		/* 7-bit */
493		*wc = c;
494		return (1);
495	}
496
497	/*
498	 * All EUC variants reserve 0xa1-0xff to identify CS1, which
499	 * is always two bytes wide.  Note that unused CS will be zero,
500	 * and that cannot be true because we know that the high order
501	 * bit must be set.
502	 */
503	if (c >= 0xa1) {
504		width = 2;
505	} else if (c == cs2) {
506		width = cs2width;
507	} else if (c == cs3) {
508		width = cs3width;
509	}
510
511	if ((int)n < width) {
512		werr("incomplete character sequence (%s)", show_mb(mb));
513		return (-1);
514	}
515
516	for (i = 1; i < width; i++) {
517		/* pull in the next byte */
518		c <<= 8;
519		c |= (uint8_t)(mb[i]);
520	}
521
522	*wc = c;
523	return (width);
524}
525
526/*
527 * EUC-CN encodes as follows:
528 *
529 * Code set 0 (ASCII):				0x21-0x7E
530 * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
531 * Code set 2:					unused
532 * Code set 3:					unused
533 */
534int
535towide_euccn(wchar_t *wc, const char *mb, unsigned n)
536{
537	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
538}
539
540/*
541 * EUC-JP encodes as follows:
542 *
543 * Code set 0 (ASCII or JIS X 0201-1976 Roman):	0x21-0x7E
544 * Code set 1 (JIS X 0208):			0xA1A1-0xFEFE
545 * Code set 2 (half-width katakana):		0x8EA1-0x8EDF
546 * Code set 3 (JIS X 0212-1990):		0x8FA1A1-0x8FFEFE
547 */
548int
549towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
550{
551	return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
552}
553
554/*
555 * EUC-KR encodes as follows:
556 *
557 * Code set 0 (ASCII or KS C 5636-1993):	0x21-0x7E
558 * Code set 1 (KS C 5601-1992):			0xA1A1-0xFEFE
559 * Code set 2:					unused
560 * Code set 3:					unused
561 */
562int
563towide_euckr(wchar_t *wc, const char *mb, unsigned n)
564{
565	return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
566}
567
568/*
569 * EUC-TW encodes as follows:
570 *
571 * Code set 0 (ASCII):				0x21-0x7E
572 * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
573 * Code set 2 (CNS 11643-1992 Planes 1-16):	0x8EA1A1A1-0x8EB0FEFE
574 * Code set 3:					unused
575 */
576int
577towide_euctw(wchar_t *wc, const char *mb, unsigned n)
578{
579	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
580}
581
582/*
583 * Public entry points.
584 */
585
586int
587to_wide(wchar_t *wc, const char *mb)
588{
589	/* this won't fail hard */
590	return (_towide(wc, mb, strlen(mb)));
591}
592
593int
594to_mb(char *mb, wchar_t wc)
595{
596	int	rv;
597
598	if ((rv = _tomb(mb, wc)) < 0) {
599		warn("%s", widemsg);
600		free(widemsg);
601		widemsg = NULL;
602	}
603	return (rv);
604}
605
606char *
607to_mb_string(const wchar_t *wcs)
608{
609	char	*mbs;
610	char	*ptr;
611	int	len;
612
613	mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
614	if (mbs == NULL) {
615		warn("out of memory");
616		return (NULL);
617	}
618	ptr = mbs;
619	while (*wcs) {
620		if ((len = to_mb(ptr, *wcs)) < 0) {
621			INTERR;
622			free(mbs);
623			return (NULL);
624		}
625		wcs++;
626		ptr += len;
627	}
628	*ptr = 0;
629	return (mbs);
630}
631
632void
633set_wide_encoding(const char *encoding)
634{
635	int i;
636
637	_towide = towide_none;
638	_tomb = tomb_none;
639	_nbits = 8;
640
641	snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s",
642	    encoding);
643	for (i = 0; mb_encodings[i].name; i++) {
644		if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
645			_towide = mb_encodings[i].towide;
646			_tomb = mb_encodings[i].tomb;
647			_encoding = mb_encodings[i].cname;
648			_nbits = mb_encodings[i].nbits;
649			break;
650		}
651	}
652}
653
654const char *
655get_wide_encoding(void)
656{
657	return (_encoding);
658}
659
660int
661max_wide(void)
662{
663	return ((int)((1U << _nbits) - 1));
664}
665