1247738Sbapt#include <yaml.h> 2247738Sbapt 3247738SbaptYAML_DECLARE(int) 4247738Sbaptyaml_parser_update_buffer(yaml_parser_t *parser, size_t length); 5247738Sbapt 6247738Sbapt#include <stdlib.h> 7247738Sbapt#include <stdio.h> 8247738Sbapt 9247738Sbapt#ifdef NDEBUG 10247738Sbapt#undef NDEBUG 11247738Sbapt#endif 12247738Sbapt#include <assert.h> 13247738Sbapt 14247738Sbapt/* 15247738Sbapt * Test cases are stolen from 16247738Sbapt * http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt 17247738Sbapt */ 18247738Sbapt 19247738Sbapttypedef struct { 20247738Sbapt char *title; 21247738Sbapt char *test; 22247738Sbapt int result; 23247738Sbapt} test_case; 24247738Sbapt 25247738Sbapttest_case utf8_sequences[] = { 26247738Sbapt /* {"title", "test 1|test 2|...|test N!", (0 or 1)}, */ 27247738Sbapt 28247738Sbapt {"a simple test", "'test' is '\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb2\xd0\xb5\xd1\x80\xd0\xba\xd0\xb0' in Russian!", 1}, 29247738Sbapt {"an empty line", "!", 1}, 30247738Sbapt 31247738Sbapt {"u-0 is a control character", "\x00!", 0}, 32247738Sbapt {"u-80 is a control character", "\xc2\x80!", 0}, 33247738Sbapt {"u-800 is valid", "\xe0\xa0\x80!", 1}, 34247738Sbapt {"u-10000 is valid", "\xf0\x90\x80\x80!", 1}, 35247738Sbapt {"5 bytes sequences are not allowed", "\xf8\x88\x80\x80\x80!", 0}, 36247738Sbapt {"6 bytes sequences are not allowed", "\xfc\x84\x80\x80\x80\x80!", 0}, 37247738Sbapt 38247738Sbapt {"u-7f is a control character", "\x7f!", 0}, 39247738Sbapt {"u-7FF is valid", "\xdf\xbf!", 1}, 40247738Sbapt {"u-FFFF is a control character", "\xef\xbf\xbf!", 0}, 41247738Sbapt {"u-1FFFFF is too large", "\xf7\xbf\xbf\xbf!", 0}, 42247738Sbapt {"u-3FFFFFF is 5 bytes", "\xfb\xbf\xbf\xbf\xbf!", 0}, 43247738Sbapt {"u-7FFFFFFF is 6 bytes", "\xfd\xbf\xbf\xbf\xbf\xbf!", 0}, 44247738Sbapt 45247738Sbapt {"u-D7FF", "\xed\x9f\xbf!", 1}, 46247738Sbapt {"u-E000", "\xee\x80\x80!", 1}, 47247738Sbapt {"u-FFFD", "\xef\xbf\xbd!", 1}, 48247738Sbapt {"u-10FFFF", "\xf4\x8f\xbf\xbf!", 1}, 49247738Sbapt {"u-110000", "\xf4\x90\x80\x80!", 0}, 50247738Sbapt 51247738Sbapt {"first continuation byte", "\x80!", 0}, 52247738Sbapt {"last continuation byte", "\xbf!", 0}, 53247738Sbapt 54247738Sbapt {"2 continuation bytes", "\x80\xbf!", 0}, 55247738Sbapt {"3 continuation bytes", "\x80\xbf\x80!", 0}, 56247738Sbapt {"4 continuation bytes", "\x80\xbf\x80\xbf!", 0}, 57247738Sbapt {"5 continuation bytes", "\x80\xbf\x80\xbf\x80!", 0}, 58247738Sbapt {"6 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf!", 0}, 59247738Sbapt {"7 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf\x80!", 0}, 60247738Sbapt 61247738Sbapt {"sequence of all 64 possible continuation bytes", 62247738Sbapt "\x80|\x81|\x82|\x83|\x84|\x85|\x86|\x87|\x88|\x89|\x8a|\x8b|\x8c|\x8d|\x8e|\x8f|" 63247738Sbapt "\x90|\x91|\x92|\x93|\x94|\x95|\x96|\x97|\x98|\x99|\x9a|\x9b|\x9c|\x9d|\x9e|\x9f|" 64247738Sbapt "\xa0|\xa1|\xa2|\xa3|\xa4|\xa5|\xa6|\xa7|\xa8|\xa9|\xaa|\xab|\xac|\xad|\xae|\xaf|" 65247738Sbapt "\xb0|\xb1|\xb2|\xb3|\xb4|\xb5|\xb6|\xb7|\xb8|\xb9|\xba|\xbb|\xbc|\xbd|\xbe|\xbf!", 0}, 66247738Sbapt {"32 first bytes of 2-byte sequences {0xc0-0xdf}", 67247738Sbapt "\xc0 |\xc1 |\xc2 |\xc3 |\xc4 |\xc5 |\xc6 |\xc7 |\xc8 |\xc9 |\xca |\xcb |\xcc |\xcd |\xce |\xcf |" 68247738Sbapt "\xd0 |\xd1 |\xd2 |\xd3 |\xd4 |\xd5 |\xd6 |\xd7 |\xd8 |\xd9 |\xda |\xdb |\xdc |\xdd |\xde |\xdf !", 0}, 69247738Sbapt {"16 first bytes of 3-byte sequences {0xe0-0xef}", 70247738Sbapt "\xe0 |\xe1 |\xe2 |\xe3 |\xe4 |\xe5 |\xe6 |\xe7 |\xe8 |\xe9 |\xea |\xeb |\xec |\xed |\xee |\xef !", 0}, 71247738Sbapt {"8 first bytes of 4-byte sequences {0xf0-0xf7}", "\xf0 |\xf1 |\xf2 |\xf3 |\xf4 |\xf5 |\xf6 |\xf7 !", 0}, 72247738Sbapt {"4 first bytes of 5-byte sequences {0xf8-0xfb}", "\xf8 |\xf9 |\xfa |\xfb !", 0}, 73247738Sbapt {"2 first bytes of 6-byte sequences {0xfc-0xfd}", "\xfc |\xfd !", 0}, 74247738Sbapt 75247738Sbapt {"sequences with last byte missing {u-0}", 76247738Sbapt "\xc0|\xe0\x80|\xf0\x80\x80|\xf8\x80\x80\x80|\xfc\x80\x80\x80\x80!", 0}, 77247738Sbapt {"sequences with last byte missing {u-...FF}", 78247738Sbapt "\xdf|\xef\xbf|\xf7\xbf\xbf|\xfb\xbf\xbf\xbf|\xfd\xbf\xbf\xbf\xbf!", 0}, 79247738Sbapt 80247738Sbapt {"impossible bytes", "\xfe|\xff|\xfe\xfe\xff\xff!", 0}, 81247738Sbapt 82247738Sbapt {"overlong sequences {u-2f}", 83247738Sbapt "\xc0\xaf|\xe0\x80\xaf|\xf0\x80\x80\xaf|\xf8\x80\x80\x80\xaf|\xfc\x80\x80\x80\x80\xaf!", 0}, 84247738Sbapt 85247738Sbapt {"maximum overlong sequences", 86247738Sbapt "\xc1\xbf|\xe0\x9f\xbf|\xf0\x8f\xbf\xbf|\xf8\x87\xbf\xbf\xbf|\xfc\x83\xbf\xbf\xbf\xbf!", 0}, 87247738Sbapt 88247738Sbapt {"overlong representation of the NUL character", 89247738Sbapt "\xc0\x80|\xe0\x80\x80|\xf0\x80\x80\x80|\xf8\x80\x80\x80\x80|\xfc\x80\x80\x80\x80\x80!", 0}, 90247738Sbapt 91247738Sbapt {"single UTF-16 surrogates", 92247738Sbapt "\xed\xa0\x80|\xed\xad\xbf|\xed\xae\x80|\xed\xaf\xbf|\xed\xb0\x80|\xed\xbe\x80|\xed\xbf\xbf!", 0}, 93247738Sbapt 94247738Sbapt {"paired UTF-16 surrogates", 95247738Sbapt "\xed\xa0\x80\xed\xb0\x80|\xed\xa0\x80\xed\xbf\xbf|\xed\xad\xbf\xed\xb0\x80|" 96247738Sbapt "\xed\xad\xbf\xed\xbf\xbf|\xed\xae\x80\xed\xb0\x80|\xed\xae\x80\xed\xbf\xbf|" 97247738Sbapt "\xed\xaf\xbf\xed\xb0\x80|\xed\xaf\xbf\xed\xbf\xbf!", 0}, 98247738Sbapt 99247738Sbapt {"other illegal code positions", "\xef\xbf\xbe|\xef\xbf\xbf!", 0}, 100247738Sbapt 101247738Sbapt {NULL, NULL, 0} 102247738Sbapt}; 103247738Sbapt 104247738Sbapttest_case boms[] = { 105247738Sbapt 106247738Sbapt /* {"title", "test!", lenth}, */ 107247738Sbapt 108247738Sbapt {"no bom (utf-8)", "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, 109247738Sbapt {"bom (utf-8)", "\xef\xbb\xbfHi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, 110247738Sbapt {"bom (utf-16-le)", "\xff\xfeH\x00i\x00 \x00i\x00s\x00 \x00\x1f\x04@\x04""8\x04""2\x04""5\x04""B\x04!", 13}, 111247738Sbapt {"bom (utf-16-be)", "\xfe\xff\x00H\x00i\x00 \x00i\x00s\x00 \x04\x1f\x04@\x04""8\x04""2\x04""5\x04""B!", 13}, 112247738Sbapt {NULL, NULL, 0} 113247738Sbapt}; 114247738Sbapt 115247738Sbaptchar *bom_original = "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82"; 116247738Sbapt 117247738Sbaptint check_utf8_sequences(void) 118247738Sbapt{ 119247738Sbapt yaml_parser_t parser; 120247738Sbapt int failed = 0; 121247738Sbapt int k; 122247738Sbapt printf("checking utf-8 sequences...\n"); 123247738Sbapt for (k = 0; utf8_sequences[k].test; k++) { 124247738Sbapt char *title = utf8_sequences[k].title; 125247738Sbapt int check = utf8_sequences[k].result; 126247738Sbapt int result; 127247738Sbapt char *start = utf8_sequences[k].test; 128247738Sbapt char *end = start; 129247738Sbapt printf("\t%s:\n", title); 130247738Sbapt while(1) { 131247738Sbapt while (*end != '|' && *end != '!') end++; 132247738Sbapt yaml_parser_initialize(&parser); 133247738Sbapt yaml_parser_set_input_string(&parser, (unsigned char *)start, end-start); 134247738Sbapt result = yaml_parser_update_buffer(&parser, end-start); 135247738Sbapt if (result != check) { 136247738Sbapt printf("\t\t- "); 137247738Sbapt failed ++; 138247738Sbapt } 139247738Sbapt else { 140247738Sbapt printf("\t\t+ "); 141247738Sbapt } 142247738Sbapt if (!parser.error) { 143247738Sbapt printf("(no error)\n"); 144247738Sbapt } 145247738Sbapt else if (parser.error == YAML_READER_ERROR) { 146247738Sbapt if (parser.problem_value != -1) { 147247738Sbapt printf("(reader error: %s: #%X at %d)\n", 148247738Sbapt parser.problem, parser.problem_value, parser.problem_offset); 149247738Sbapt } 150247738Sbapt else { 151247738Sbapt printf("(reader error: %s at %d)\n", 152247738Sbapt parser.problem, parser.problem_offset); 153247738Sbapt } 154247738Sbapt } 155247738Sbapt if (*end == '!') break; 156247738Sbapt start = ++end; 157247738Sbapt yaml_parser_delete(&parser); 158247738Sbapt }; 159247738Sbapt printf("\n"); 160247738Sbapt } 161247738Sbapt printf("checking utf-8 sequences: %d fail(s)\n", failed); 162247738Sbapt return failed; 163247738Sbapt} 164247738Sbapt 165247738Sbaptint check_boms(void) 166247738Sbapt{ 167247738Sbapt yaml_parser_t parser; 168247738Sbapt int failed = 0; 169247738Sbapt int k; 170247738Sbapt printf("checking boms...\n"); 171247738Sbapt for (k = 0; boms[k].test; k++) { 172247738Sbapt char *title = boms[k].title; 173247738Sbapt int check = boms[k].result; 174247738Sbapt int result; 175247738Sbapt char *start = boms[k].test; 176247738Sbapt char *end = start; 177247738Sbapt while (*end != '!') end++; 178247738Sbapt printf("\t%s: ", title); 179247738Sbapt yaml_parser_initialize(&parser); 180247738Sbapt yaml_parser_set_input_string(&parser, (unsigned char *)start, end-start); 181247738Sbapt result = yaml_parser_update_buffer(&parser, end-start); 182247738Sbapt if (!result) { 183247738Sbapt printf("- (reader error: %s at %d)\n", parser.problem, parser.problem_offset); 184247738Sbapt failed++; 185247738Sbapt } 186247738Sbapt else { 187247738Sbapt if (parser.unread != check) { 188247738Sbapt printf("- (length=%d while expected length=%d)\n", parser.unread, check); 189247738Sbapt failed++; 190247738Sbapt } 191247738Sbapt else if (memcmp(parser.buffer.start, bom_original, check) != 0) { 192247738Sbapt printf("- (value '%s' does not equal to the original value '%s')\n", parser.buffer.start, bom_original); 193247738Sbapt failed++; 194247738Sbapt } 195247738Sbapt else { 196247738Sbapt printf("+\n"); 197247738Sbapt } 198247738Sbapt } 199247738Sbapt yaml_parser_delete(&parser); 200247738Sbapt } 201247738Sbapt printf("checking boms: %d fail(s)\n", failed); 202247738Sbapt return failed; 203247738Sbapt} 204247738Sbapt 205247738Sbapt#define LONG 100000 206247738Sbapt 207247738Sbaptint check_long_utf8(void) 208247738Sbapt{ 209247738Sbapt yaml_parser_t parser; 210247738Sbapt int k = 0; 211247738Sbapt int j; 212247738Sbapt int failed = 0; 213247738Sbapt unsigned char ch0, ch1; 214247738Sbapt unsigned char *buffer = malloc(3+LONG*2); 215247738Sbapt assert(buffer); 216247738Sbapt printf("checking a long utf8 sequence...\n"); 217247738Sbapt buffer[k++] = '\xef'; 218247738Sbapt buffer[k++] = '\xbb'; 219247738Sbapt buffer[k++] = '\xbf'; 220247738Sbapt for (j = 0; j < LONG; j ++) { 221247738Sbapt if (j % 2) { 222247738Sbapt buffer[k++] = '\xd0'; 223247738Sbapt buffer[k++] = '\x90'; 224247738Sbapt } 225247738Sbapt else { 226247738Sbapt buffer[k++] = '\xd0'; 227247738Sbapt buffer[k++] = '\xaf'; 228247738Sbapt } 229247738Sbapt } 230247738Sbapt yaml_parser_initialize(&parser); 231247738Sbapt yaml_parser_set_input_string(&parser, buffer, 3+LONG*2); 232247738Sbapt for (k = 0; k < LONG; k++) { 233247738Sbapt if (!parser.unread) { 234247738Sbapt if (!yaml_parser_update_buffer(&parser, 1)) { 235247738Sbapt printf("\treader error: %s at %d\n", parser.problem, parser.problem_offset); 236247738Sbapt failed = 1; 237247738Sbapt break; 238247738Sbapt } 239247738Sbapt } 240247738Sbapt if (!parser.unread) { 241247738Sbapt printf("\tnot enough characters at %d\n", k); 242247738Sbapt failed = 1; 243247738Sbapt break; 244247738Sbapt } 245247738Sbapt if (k % 2) { 246247738Sbapt ch0 = '\xd0'; 247247738Sbapt ch1 = '\x90'; 248247738Sbapt } 249247738Sbapt else { 250247738Sbapt ch0 = '\xd0'; 251247738Sbapt ch1 = '\xaf'; 252247738Sbapt } 253247738Sbapt if (parser.buffer.pointer[0] != ch0 || parser.buffer.pointer[1] != ch1) { 254247738Sbapt printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n", 255247738Sbapt (int)parser.buffer.pointer[0], (int)parser.buffer.pointer[1], 256247738Sbapt (int)ch0, (int)ch1); 257247738Sbapt failed = 1; 258247738Sbapt break; 259247738Sbapt } 260247738Sbapt parser.buffer.pointer += 2; 261247738Sbapt parser.unread -= 1; 262247738Sbapt } 263247738Sbapt if (!failed) { 264247738Sbapt if (!yaml_parser_update_buffer(&parser, 1)) { 265247738Sbapt printf("\treader error: %s at %d\n", parser.problem, parser.problem_offset); 266247738Sbapt failed = 1; 267247738Sbapt } 268247738Sbapt else if (parser.buffer.pointer[0] != '\0') { 269247738Sbapt printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser.buffer.pointer[0], parser.eof, parser.unread); 270247738Sbapt failed = 1; 271247738Sbapt } 272247738Sbapt } 273247738Sbapt yaml_parser_delete(&parser); 274247738Sbapt free(buffer); 275247738Sbapt printf("checking a long utf8 sequence: %d fail(s)\n", failed); 276247738Sbapt return failed; 277247738Sbapt} 278247738Sbapt 279247738Sbaptint check_long_utf16(void) 280247738Sbapt{ 281247738Sbapt yaml_parser_t parser; 282247738Sbapt int k = 0; 283247738Sbapt int j; 284247738Sbapt int failed = 0; 285247738Sbapt unsigned char ch0, ch1; 286247738Sbapt unsigned char *buffer = malloc(2+LONG*2); 287247738Sbapt assert(buffer); 288247738Sbapt printf("checking a long utf16 sequence...\n"); 289247738Sbapt buffer[k++] = '\xff'; 290247738Sbapt buffer[k++] = '\xfe'; 291247738Sbapt for (j = 0; j < LONG; j ++) { 292247738Sbapt if (j % 2) { 293247738Sbapt buffer[k++] = '\x10'; 294247738Sbapt buffer[k++] = '\x04'; 295247738Sbapt } 296247738Sbapt else { 297247738Sbapt buffer[k++] = '/'; 298247738Sbapt buffer[k++] = '\x04'; 299247738Sbapt } 300247738Sbapt } 301247738Sbapt yaml_parser_initialize(&parser); 302247738Sbapt yaml_parser_set_input_string(&parser, buffer, 2+LONG*2); 303247738Sbapt for (k = 0; k < LONG; k++) { 304247738Sbapt if (!parser.unread) { 305247738Sbapt if (!yaml_parser_update_buffer(&parser, 1)) { 306247738Sbapt printf("\treader error: %s at %d\n", parser.problem, parser.problem_offset); 307247738Sbapt failed = 1; 308247738Sbapt break; 309247738Sbapt } 310247738Sbapt } 311247738Sbapt if (!parser.unread) { 312247738Sbapt printf("\tnot enough characters at %d\n", k); 313247738Sbapt failed = 1; 314247738Sbapt break; 315247738Sbapt } 316247738Sbapt if (k % 2) { 317247738Sbapt ch0 = '\xd0'; 318247738Sbapt ch1 = '\x90'; 319247738Sbapt } 320247738Sbapt else { 321247738Sbapt ch0 = '\xd0'; 322247738Sbapt ch1 = '\xaf'; 323247738Sbapt } 324247738Sbapt if (parser.buffer.pointer[0] != ch0 || parser.buffer.pointer[1] != ch1) { 325247738Sbapt printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n", 326247738Sbapt (int)parser.buffer.pointer[0], (int)parser.buffer.pointer[1], 327247738Sbapt (int)ch0, (int)ch1); 328247738Sbapt failed = 1; 329247738Sbapt break; 330247738Sbapt } 331247738Sbapt parser.buffer.pointer += 2; 332247738Sbapt parser.unread -= 1; 333247738Sbapt } 334247738Sbapt if (!failed) { 335247738Sbapt if (!yaml_parser_update_buffer(&parser, 1)) { 336247738Sbapt printf("\treader error: %s at %d\n", parser.problem, parser.problem_offset); 337247738Sbapt failed = 1; 338247738Sbapt } 339247738Sbapt else if (parser.buffer.pointer[0] != '\0') { 340247738Sbapt printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser.buffer.pointer[0], parser.eof, parser.unread); 341247738Sbapt failed = 1; 342247738Sbapt } 343247738Sbapt } 344247738Sbapt yaml_parser_delete(&parser); 345247738Sbapt free(buffer); 346247738Sbapt printf("checking a long utf16 sequence: %d fail(s)\n", failed); 347247738Sbapt return failed; 348247738Sbapt} 349247738Sbapt 350247738Sbaptint 351247738Sbaptmain(void) 352247738Sbapt{ 353247738Sbapt return check_utf8_sequences() + check_boms() + check_long_utf8() + check_long_utf16(); 354247738Sbapt} 355