Invalid UTF-8
William Spitzak <[email protected]>
| Newsgroups | gmane.text.yaml.general |
|---|---|
| Message-ID | <[email protected]> |
I'm sure this has come up before, but anyway: In our software it looks like YAML would be a great way to store are retrieve data. However we have the need to store arbitrary bytes, with the caveat that they are *LIKELY* to be UTF-8. Examples are Unix filenames, and metadata stored in image files such as comments. This requires that we be able to losslessly store invalid UTF-8. The attached patch against yaml-0.1.2 implements these changes: 1. Invalid UTF-8 is stored by writing each erroneous byte using the new escape sequence "\XNN" in quoted strings. Thus the saved files are themselves valid UTF-8 and writing UTF-16 files still works. 2. The parser reads "\XNN" sequences in quoted strings. It can also read raw invalid UTF-8 bytes from the input file without changing it. 3. Mismatched surrogate pairs (ie invalid UTF-16) are accepted and read and written directly. This means that a UTF-16 file read/written by YAML can contain raw invalid UTF-16 sequences. The obvious encoding of invalid UTF-16 to UTF-8 is used to read/write UTF-8 files. Notice that this encoding is lossless and does not interfere with correct UTF-8 encoding of Unicode characters >= U+10000. 4. The parser also accepts mismatched surrogate pairs as "\uNNNN" escapes in quoted strings. It may be desirable to write invalid surrogate pairs this way but I did not implement this. 5. I commented out the production of "\xNN" escapes as that indicates a raw byte, not Unicode, for C and C++. It writes "\u00NN" escapes instead. This is not a requirement however. 6. It also accepts arbitrary strings in tag names. It %-encodes all the bytes in any valid or invalid UTF-8 encoding. This matches how invalid UTF-8 is handled in URLs by most servers. NOTES ON THE IMPLEMENTATION: Handling UTF-8 is MUCH easier than most people believe. It is far more useful to think of it as a byte stream and to use byte operations than to try to decode it. I tried to make this patch do this, while minimizing the size of the diff. An "error" is ONE byte long and that byte ALWAYS has the high bit set. A pointer to an error can NEVER match any ASCII character or any valid UTF-8 encoding. This means search can be byte oriented and unconcerned about errors. If you are pointing at a UTF-8 character or error and you move one byte, you will either be pointing at an error or at the next character. This makes searching for UTF-8 trivial by doing byte pattern matching, there is no need to decode UTF-8 or detect errors. For instance you can find the next BREAK by running IS_BREAK every byte. Other than BREAK and BOM, the only characters handled specially by YAML are one byte long. This means the majority of it can treat files as byte streams and not decode anything. I initially replaced all the macros like READ() with READ1() and READN() replacements. However I then renamed READ1() back to READ() as this greatly reduced the diff size. IS_BREAK is somewhat annoying. I think it may be a good idea to redefine breaks as only being NL Bill Spitzak Rhythm & Hues Software Department ------------------------------------------------------------------------------ Let Crystal Reports handle the reporting - Free Crystal Reports 2008 30-Day trial. Simplify your report design, integration and deployment - and focus on what you do best, core application coding. Discover what's new with Crystal Reports now. http://p.sf.net/sfu/bobj-july _______________________________________________ Yaml-core mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/yaml-core
patch
(text/plain, 36.3 KB)
diff -ur original_src/api.c src/api.c
--- original_src/api.c 2008-12-27 15:18:21.000000000 -0800
+++ src/api.c 2009-08-18 18:18:41.000000000 -0700
@@ -611,51 +611,6 @@
}
/*
- * Check if a string is a valid UTF-8 sequence.
- *
- * Check 'reader.c' for more details on UTF-8 encoding.
- */
-
-static int
-yaml_check_utf8(yaml_char_t *start, size_t length)
-{
- yaml_char_t *end = start+length;
- yaml_char_t *pointer = start;
-
- while (pointer < end) {
- unsigned char octet;
- unsigned int width;
- unsigned int value;
- size_t k;
-
- octet = pointer[0];
- width = (octet & 0x80) == 0x00 ? 1 :
- (octet & 0xE0) == 0xC0 ? 2 :
- (octet & 0xF0) == 0xE0 ? 3 :
- (octet & 0xF8) == 0xF0 ? 4 : 0;
- value = (octet & 0x80) == 0x00 ? octet & 0x7F :
- (octet & 0xE0) == 0xC0 ? octet & 0x1F :
- (octet & 0xF0) == 0xE0 ? octet & 0x0F :
- (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
- if (!width) return 0;
- if (pointer+width > end) return 0;
- for (k = 1; k < width; k ++) {
- octet = pointer[k];
- if ((octet & 0xC0) != 0x80) return 0;
- value = (value << 6) + (octet & 0x3F);
- }
- if (!((width == 1) ||
- (width == 2 && value >= 0x80) ||
- (width == 3 && value >= 0x800) ||
- (width == 4 && value >= 0x10000))) return 0;
-
- pointer += width;
- }
-
- return 1;
-}
-
-/*
* Create STREAM-START.
*/
@@ -731,12 +686,6 @@
tag_directive != tag_directives_end; tag_directive ++) {
assert(tag_directive->handle);
assert(tag_directive->prefix);
- if (!yaml_check_utf8(tag_directive->handle,
- strlen((char *)tag_directive->handle)))
- goto error;
- if (!yaml_check_utf8(tag_directive->prefix,
- strlen((char *)tag_directive->prefix)))
- goto error;
value.handle = yaml_strdup(tag_directive->handle);
value.prefix = yaml_strdup(tag_directive->prefix);
if (!value.handle || !value.prefix) goto error;
@@ -796,8 +745,6 @@
assert(event); /* Non-NULL event object is expected. */
assert(anchor); /* Non-NULL anchor is expected. */
- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0;
-
anchor_copy = yaml_strdup(anchor);
if (!anchor_copy)
return 0;
@@ -827,13 +774,11 @@
assert(value); /* Non-NULL anchor is expected. */
if (anchor) {
- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error;
anchor_copy = yaml_strdup(anchor);
if (!anchor_copy) goto error;
}
if (tag) {
- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
tag_copy = yaml_strdup(tag);
if (!tag_copy) goto error;
}
@@ -842,7 +787,6 @@
length = strlen((char *)value);
}
- if (!yaml_check_utf8(value, length)) goto error;
value_copy = yaml_malloc(length+1);
if (!value_copy) goto error;
memcpy(value_copy, value, length);
@@ -877,13 +821,11 @@
assert(event); /* Non-NULL event object is expected. */
if (anchor) {
- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error;
anchor_copy = yaml_strdup(anchor);
if (!anchor_copy) goto error;
}
if (tag) {
- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
tag_copy = yaml_strdup(tag);
if (!tag_copy) goto error;
}
@@ -932,13 +874,11 @@
assert(event); /* Non-NULL event object is expected. */
if (anchor) {
- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error;
anchor_copy = yaml_strdup(anchor);
if (!anchor_copy) goto error;
}
if (tag) {
- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
tag_copy = yaml_strdup(tag);
if (!tag_copy) goto error;
}
@@ -1072,12 +1012,6 @@
tag_directive != tag_directives_end; tag_directive ++) {
assert(tag_directive->handle);
assert(tag_directive->prefix);
- if (!yaml_check_utf8(tag_directive->handle,
- strlen((char *)tag_directive->handle)))
- goto error;
- if (!yaml_check_utf8(tag_directive->prefix,
- strlen((char *)tag_directive->prefix)))
- goto error;
value.handle = yaml_strdup(tag_directive->handle);
value.prefix = yaml_strdup(tag_directive->prefix);
if (!value.handle || !value.prefix) goto error;
@@ -1210,7 +1144,6 @@
tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG;
}
- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
tag_copy = yaml_strdup(tag);
if (!tag_copy) goto error;
@@ -1218,7 +1151,6 @@
length = strlen((char *)value);
}
- if (!yaml_check_utf8(value, length)) goto error;
value_copy = yaml_malloc(length+1);
if (!value_copy) goto error;
memcpy(value_copy, value, length);
@@ -1262,7 +1194,6 @@
tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG;
}
- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
tag_copy = yaml_strdup(tag);
if (!tag_copy) goto error;
@@ -1307,7 +1238,6 @@
tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG;
}
- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
tag_copy = yaml_strdup(tag);
if (!tag_copy) goto error;
diff -ur original_src/emitter.c src/emitter.c
--- original_src/emitter.c 2008-12-27 15:18:21.000000000 -0800
+++ src/emitter.c 2009-08-18 17:50:47.000000000 -0700
@@ -37,7 +37,7 @@
1))
/*
- * Copy a character from a string into buffer.
+ * Copy a byte from a string into buffer.
*/
#define WRITE(emitter,string) \
@@ -46,6 +46,12 @@
emitter->column ++, \
1))
+#define WRITEN(emitter,string,n) \
+ (FLUSH(emitter) \
+ && (COPYN(emitter->buffer,string,n), \
+ emitter->column ++, \
+ 1))
+
/*
* Copy a line break character from a string into buffer.
*/
@@ -56,7 +62,7 @@
(PUT_BREAK(emitter), \
string.pointer ++, \
1) : \
- (COPY(emitter->buffer,string), \
+ (COPYN(emitter->buffer,string,WIDTH(string)), \
emitter->column = 0, \
emitter->line ++, \
1)))
@@ -1458,6 +1464,57 @@
return 1;
}
+/**
+ Return the number of bytes in the next UTF-8 character.
+
+ Returns 0 for any error, includinge incorrect bytes, not enough
+ bytes before the end pointer, and overlong encodings. If 0 is
+ returned, pointer[0] will always have the high bit set and will
+ thus never match any ASCII character.
+
+ The encodings of surrogate halves are allowed! Otherwise it is not
+ possible to losslessly encode invalid UTF-16 into UTF-8. (There is
+ a vocal contingent trying to sabotage UTF-8 by declaring surrogate
+ half encodings invalid. Anybody such claim should be investigated
+ carefully: if that user's code does not also reject invalid UTF-16,
+ then they are being hypocrites and can be ignored.)
+*/
+static unsigned int utf8_width(yaml_char_t* pointer, yaml_char_t* end)
+{
+ unsigned char octet = pointer[0];
+ if (octet < 0x80) {
+ /* 1-byte character */
+ return 1;
+ } else if (octet < 0xC2) {
+ /* continuation byte or overlong 2-byte encoding */
+ return 0;
+ } else if (octet < 0xE0) {
+ /* 2-byte character */
+ if (end-pointer < 2) return 0;
+ if ((pointer[1] & 0xC0) != 0x80) return 0;
+ return 2;
+ } else if (octet < 0xF0) {
+ /* 3-byte character */
+ if (end-pointer < 3) return 0;
+ if (octet == 0xE0 && pointer[1] < 0xA0) return 0; /* overlong */
+ if ((pointer[1] & 0xC0) != 0x80) return 0;
+ if ((pointer[2] & 0xC0) != 0x80) return 0;
+ return 3;
+ } else if (octet < 0xF5) {
+ /* 4-byte character */
+ if (end-pointer < 4) return 0;
+ if (octet == 0xF0 && pointer[1] < 0x90) return 0; /* overlong */
+ if (octet == 0xF4 && pointer[1] > 0x8F) return 0; /* > 0x10FFFF */
+ if ((pointer[1] & 0xC0) != 0x80) return 0;
+ if ((pointer[2] & 0xC0) != 0x80) return 0;
+ if ((pointer[3] & 0xC0) != 0x80) return 0;
+ return 4;
+ } else {
+ /* can never appear in UTF-8 */
+ return 0;
+ }
+}
+
/*
* Check if a scalar is valid.
*/
@@ -1481,7 +1538,6 @@
int space_break = 0;
int preceeded_by_whitespace = 0;
- int followed_by_whitespace = 0;
int previous_space = 0;
int previous_break = 0;
@@ -1510,10 +1566,17 @@
}
preceeded_by_whitespace = 1;
- followed_by_whitespace = IS_BLANKZ_AT(string, WIDTH(string));
while (string.pointer != string.end)
{
+ unsigned int width = utf8_width(string.pointer, string.end);
+
+ if (!width) {
+ special_characters = 1;
+ string.pointer++;
+ continue;
+ }
+
if (string.start == string.pointer)
{
if (CHECK(string, '#') || CHECK(string, ',')
@@ -1530,12 +1593,12 @@
if (CHECK(string, '?') || CHECK(string, ':')) {
flow_indicators = 1;
- if (followed_by_whitespace) {
+ if (IS_BLANKZ_AT(string, 1)) {
block_indicators = 1;
}
}
- if (CHECK(string, '-') && followed_by_whitespace) {
+ if (CHECK(string, '-') && IS_BLANKZ_AT(string, 1)) {
flow_indicators = 1;
block_indicators = 1;
}
@@ -1550,7 +1613,7 @@
if (CHECK(string, ':')) {
flow_indicators = 1;
- if (followed_by_whitespace) {
+ if (IS_BLANKZ_AT(string, 1)) {
block_indicators = 1;
}
}
@@ -1566,10 +1629,6 @@
special_characters = 1;
}
- if (IS_BREAK(string)) {
- line_breaks = 1;
- }
-
if (IS_SPACE(string))
{
if (string.start == string.pointer) {
@@ -1586,6 +1645,7 @@
}
else if (IS_BREAK(string))
{
+ line_breaks = 1;
if (string.start == string.pointer) {
leading_break = 1;
}
@@ -1605,10 +1665,7 @@
}
preceeded_by_whitespace = IS_BLANKZ(string);
- MOVE(string);
- if (string.pointer != string.end) {
- followed_by_whitespace = IS_BLANKZ_AT(string, WIDTH(string));
- }
+ MOVEN(string, width);
}
emitter->scalar_data.multiline = line_breaks;
@@ -1851,18 +1908,15 @@
if (!WRITE(emitter, string)) return 0;
}
else {
- int width = WIDTH(string);
- unsigned int value;
- while (width --) {
- value = *(string.pointer++);
- if (!PUT(emitter, '%')) return 0;
- if (!PUT(emitter, (value >> 4)
- + ((value >> 4) < 10 ? '0' : 'A' - 10)))
- return 0;
- if (!PUT(emitter, (value & 0x0F)
- + ((value & 0x0F) < 10 ? '0' : 'A' - 10)))
- return 0;
- }
+ /* %-encode all other bytes, including valid and invalid UTF-8 */
+ unsigned char value = *(string.pointer++);
+ if (!PUT(emitter, '%')) return 0;
+ if (!PUT(emitter, (value >> 4)
+ + ((value >> 4) < 10 ? '0' : 'A' - 10)))
+ return 0;
+ if (!PUT(emitter, (value & 0x0F)
+ + ((value & 0x0F) < 10 ? '0' : 'A' - 10)))
+ return 0;
}
}
@@ -1991,6 +2045,8 @@
return 1;
}
+static unsigned char utf8_mask[5] = {0xFF, 0x7F, 0x1F, 0x0F, 0x07};
+
static int
yaml_emitter_write_double_quoted_scalar(yaml_emitter_t *emitter,
yaml_char_t *value, size_t length, int allow_breaks)
@@ -2003,27 +2059,40 @@
while (string.pointer != string.end)
{
+ unsigned int width = utf8_width(string.pointer, string.end);
+
+ if (width == 0) {
+ /*
+ UTF-8 encoding error. This is a byte with the high bit
+ set. The parser has been altered to read \XNN as a raw
+ byte. I would prefer to use lowercase x but the old
+ writer produces that for legal UTF-8 encodings of
+ U+0080..U+00FF. It is not clear what other parsers will
+ do with this, though previous libyaml threw an error.
+ */
+ unsigned int value = string.pointer[0];
+ int digit;
+ if (!PUT(emitter, '\\')) return 0;
+ if (!PUT(emitter, 'X')) return 0;
+ digit = (value >> 4) & 0x0F;
+ if (!PUT(emitter, digit + (digit < 10 ? '0' : 'A'-10))) return 0;
+ digit = value & 0x0F;
+ if (!PUT(emitter, digit + (digit < 10 ? '0' : 'A'-10))) return 0;
+ MOVE(string);
+ spaces = 0;
+ continue;
+ }
+
if (!IS_PRINTABLE(string) || (!emitter->unicode && !IS_ASCII(string))
|| IS_BOM(string) || IS_BREAK(string)
|| CHECK(string, '"') || CHECK(string, '\\'))
{
- unsigned char octet;
- unsigned int width;
unsigned int value;
int k;
- octet = string.pointer[0];
- width = (octet & 0x80) == 0x00 ? 1 :
- (octet & 0xE0) == 0xC0 ? 2 :
- (octet & 0xF0) == 0xE0 ? 3 :
- (octet & 0xF8) == 0xF0 ? 4 : 0;
- value = (octet & 0x80) == 0x00 ? octet & 0x7F :
- (octet & 0xE0) == 0xC0 ? octet & 0x1F :
- (octet & 0xF0) == 0xE0 ? octet & 0x0F :
- (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
+ value = string.pointer[0] & utf8_mask[width];
for (k = 1; k < (int)width; k ++) {
- octet = string.pointer[k];
- value = (value << 6) + (octet & 0x3F);
+ value = (value << 6) + (string.pointer[k] & 0x3F);
}
string.pointer += width;
@@ -2092,11 +2161,13 @@
break;
default:
- if (value <= 0xFF) {
+ // \xNN sequence is disabled so that it may be used in the
+ // future for invalid byte sequences
+ /*if (value <= 0xFF) {
if (!PUT(emitter, 'x')) return 0;
width = 2;
}
- else if (value <= 0xFFFF) {
+ else*/ if (value <= 0xFFFF) {
if (!PUT(emitter, 'u')) return 0;
width = 4;
}
@@ -2131,7 +2202,7 @@
}
else
{
- if (!WRITE(emitter, string)) return 0;
+ if (!WRITEN(emitter, string, width)) return 0;
spaces = 0;
}
}
diff -ur original_src/reader.c src/reader.c
--- original_src/reader.c 2008-12-27 15:18:21.000000000 -0800
+++ src/reader.c 2009-08-18 16:06:10.000000000 -0700
@@ -187,7 +187,6 @@
while (parser->raw_buffer.pointer != parser->raw_buffer.last)
{
unsigned int value = 0, value2 = 0;
- int incomplete = 0;
unsigned char octet;
unsigned int width = 0;
int low, high;
@@ -199,96 +198,22 @@
switch (parser->encoding)
{
case YAML_UTF8_ENCODING:
-
- /*
- * Decode a UTF-8 character. Check RFC 3629
- * (http://www.ietf.org/rfc/rfc3629.txt) for more details.
- *
- * The following table (taken from the RFC) is used for
- * decoding.
- *
- * Char. number range | UTF-8 octet sequence
- * (hexadecimal) | (binary)
- * --------------------+------------------------------------
- * 0000 0000-0000 007F | 0xxxxxxx
- * 0000 0080-0000 07FF | 110xxxxx 10xxxxxx
- * 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
- * 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
- *
- * Additionally, the characters in the range 0xD800-0xDFFF
- * are prohibited as they are reserved for use with UTF-16
- * surrogate pairs.
- */
-
- /* Determine the length of the UTF-8 sequence. */
-
octet = parser->raw_buffer.pointer[0];
- width = (octet & 0x80) == 0x00 ? 1 :
- (octet & 0xE0) == 0xC0 ? 2 :
- (octet & 0xF0) == 0xE0 ? 3 :
- (octet & 0xF8) == 0xF0 ? 4 : 0;
-
- /* Check if the leading octet is valid. */
-
- if (!width)
- return yaml_parser_set_reader_error(parser,
- "Invalid leading UTF-8 octet",
- parser->offset, octet);
-
- /* Check if the raw buffer contains an incomplete character. */
-
- if (width > raw_unread) {
- if (parser->eof) {
- return yaml_parser_set_reader_error(parser,
- "Incomplete UTF-8 octet sequence",
- parser->offset, -1);
- }
- incomplete = 1;
- break;
- }
-
- /* Decode the leading octet. */
-
- value = (octet & 0x80) == 0x00 ? octet & 0x7F :
- (octet & 0xE0) == 0xC0 ? octet & 0x1F :
- (octet & 0xF0) == 0xE0 ? octet & 0x0F :
- (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
-
- /* Check and decode the trailing octets. */
-
- for (k = 1; k < width; k ++)
- {
- octet = parser->raw_buffer.pointer[k];
-
- /* Check if the octet is valid. */
-
- if ((octet & 0xC0) != 0x80)
- return yaml_parser_set_reader_error(parser,
- "Invalid trailing UTF-8 octet",
- parser->offset+k, octet);
-
- /* Decode the octet. */
-
- value = (value << 6) + (octet & 0x3F);
- }
-
- /* Check the length of the sequence against the value. */
-
- if (!((width == 1) ||
- (width == 2 && value >= 0x80) ||
- (width == 3 && value >= 0x800) ||
- (width == 4 && value >= 0x10000)))
- return yaml_parser_set_reader_error(parser,
- "Invalid length of a UTF-8 sequence",
- parser->offset, -1);
-
- /* Check the range of the value. */
-
- if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
+ /*
+ We can only disallow characters without the high
+ bit set. Characters with the high bit set are required
+ for invalid UTF-8 strings to be encoded, as we
+ cannot rely on any backslash sequences working.
+ */
+ if (! (octet == 0x09 || octet == 0x0A || octet == 0x0D
+ || (octet >= 0x20 && octet != 0x7F) ))
return yaml_parser_set_reader_error(parser,
- "Invalid Unicode character",
+ "Control characters are not allowed",
parser->offset, value);
+ parser->raw_buffer.pointer++;
+ parser->offset++;
+ *(parser->buffer.last++) = octet;
break;
case YAML_UTF16LE_ENCODING:
@@ -331,7 +256,6 @@
"Incomplete UTF-16 character",
parser->offset, -1);
}
- incomplete = 1;
break;
}
@@ -339,109 +263,88 @@
value = parser->raw_buffer.pointer[low]
+ (parser->raw_buffer.pointer[high] << 8);
-
- /* Check for unexpected low surrogate area. */
-
- if ((value & 0xFC00) == 0xDC00)
- return yaml_parser_set_reader_error(parser,
- "Unexpected low surrogate area",
- parser->offset, value);
+ width = 2;
/* Check for a high surrogate area. */
if ((value & 0xFC00) == 0xD800) {
- width = 4;
-
/* Check for incomplete surrogate pair. */
if (raw_unread < 4) {
- if (parser->eof) {
- return yaml_parser_set_reader_error(parser,
- "Incomplete UTF-16 surrogate pair",
- parser->offset, -1);
+ if (parser->eof) { /* trailing high surrogate */
+ width = 2;
+ } else {
+ break; /* Can't tell until we have more raw characters */
+ }
+ } else {
+
+ /* Get the next character. */
+
+ value2 = parser->raw_buffer.pointer[low+2]
+ + (parser->raw_buffer.pointer[high+2] << 8);
+
+ /* Check for a low surrogate area. */
+ if ((value2 & 0xFC00) == 0xDC00) {
+ width = 4;
+ /* Generate the value of the surrogate pair. */
+ value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
}
- incomplete = 1;
- break;
}
+ }
- /* Get the next character. */
+ /* Check if the raw buffer contains enough bytes to form a character. */
- value2 = parser->raw_buffer.pointer[low+2]
- + (parser->raw_buffer.pointer[high+2] << 8);
+ /*
+ * Check if the character is in the allowed range:
+ * #x9 | #xA | #xD | [#x20-#x7E] (8 bit)
+ * | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit)
+ * | [#x10000-#x10FFFF] (32 bit)
+ */
+ /* Modified to allow all 16-bit values as \uNNNN may not
+ work for some parsers for these values. */
+ if (! (value == 0x09 || value == 0x0A || value == 0x0D
+ || (value >= 0x20 && value <= 0x7E)
+ || (value == 0x85) || value >= 0xA0))
+ return yaml_parser_set_reader_error(parser,
+ "Control characters are not allowed",
+ parser->offset, value);
- /* Check for a low surrogate area. */
+ /* Move the raw pointers. */
- if ((value2 & 0xFC00) != 0xDC00)
- return yaml_parser_set_reader_error(parser,
- "Expected low surrogate area",
- parser->offset+2, value2);
+ parser->raw_buffer.pointer += width;
+ parser->offset += width;
- /* Generate the value of the surrogate pair. */
+ /* Finally put the character into the buffer. */
- value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
+ /* 0000 0000-0000 007F -> 0xxxxxxx */
+ if (value <= 0x7F) {
+ *(parser->buffer.last++) = value;
}
-
+ /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
+ else if (value <= 0x7FF) {
+ *(parser->buffer.last++) = 0xC0 + (value >> 6);
+ *(parser->buffer.last++) = 0x80 + (value & 0x3F);
+ }
+ /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
+ else if (value <= 0xFFFF) {
+ *(parser->buffer.last++) = 0xE0 + (value >> 12);
+ *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
+ *(parser->buffer.last++) = 0x80 + (value & 0x3F);
+ }
+ /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
else {
- width = 2;
+ *(parser->buffer.last++) = 0xF0 + (value >> 18);
+ *(parser->buffer.last++) = 0x80 + ((value >> 12) & 0x3F);
+ *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
+ *(parser->buffer.last++) = 0x80 + (value & 0x3F);
}
-
break;
default:
assert(1); /* Impossible. */
}
- /* Check if the raw buffer contains enough bytes to form a character. */
-
- if (incomplete) break;
-
- /*
- * Check if the character is in the allowed range:
- * #x9 | #xA | #xD | [#x20-#x7E] (8 bit)
- * | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit)
- * | [#x10000-#x10FFFF] (32 bit)
- */
-
- if (! (value == 0x09 || value == 0x0A || value == 0x0D
- || (value >= 0x20 && value <= 0x7E)
- || (value == 0x85) || (value >= 0xA0 && value <= 0xD7FF)
- || (value >= 0xE000 && value <= 0xFFFD)
- || (value >= 0x10000 && value <= 0x10FFFF)))
- return yaml_parser_set_reader_error(parser,
- "Control characters are not allowed",
- parser->offset, value);
-
- /* Move the raw pointers. */
-
- parser->raw_buffer.pointer += width;
- parser->offset += width;
-
- /* Finally put the character into the buffer. */
-
- /* 0000 0000-0000 007F -> 0xxxxxxx */
- if (value <= 0x7F) {
- *(parser->buffer.last++) = value;
- }
- /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
- else if (value <= 0x7FF) {
- *(parser->buffer.last++) = 0xC0 + (value >> 6);
- *(parser->buffer.last++) = 0x80 + (value & 0x3F);
- }
- /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
- else if (value <= 0xFFFF) {
- *(parser->buffer.last++) = 0xE0 + (value >> 12);
- *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
- *(parser->buffer.last++) = 0x80 + (value & 0x3F);
- }
- /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
- else {
- *(parser->buffer.last++) = 0xF0 + (value >> 18);
- *(parser->buffer.last++) = 0x80 + ((value >> 12) & 0x3F);
- *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
- *(parser->buffer.last++) = 0x80 + (value & 0x3F);
- }
-
parser->unread ++;
}
diff -ur original_src/scanner.c src/scanner.c
--- original_src/scanner.c 2008-12-27 15:18:21.000000000 -0800
+++ src/scanner.c 2009-08-18 17:56:35.000000000 -0700
@@ -495,8 +495,13 @@
(parser->mark.index ++, \
parser->mark.column ++, \
parser->unread --, \
- parser->buffer.pointer += WIDTH(parser->buffer))
+ parser->buffer.pointer ++)
+#define SKIPN(parser,n) \
+ (parser->mark.index ++, \
+ parser->mark.column ++, \
+ parser->unread --, \
+ parser->buffer.pointer += (n))
#define SKIP_LINE(parser) \
(IS_CRLF(parser->buffer) ? \
(parser->mark.index += 2, \
@@ -523,6 +528,14 @@
parser->unread --, \
1) : 0)
+#define READN(parser,string,n) \
+ (STRING_EXTEND(parser,string) ? \
+ (COPYN(string,parser->buffer,n), \
+ parser->mark.index ++, \
+ parser->mark.column ++, \
+ parser->unread --, \
+ 1) : 0)
+
/*
* Copy a line break character to a string buffer and advance pointers.
*/
@@ -1924,7 +1937,7 @@
if (!CACHE(parser, 1)) return 0;
if (parser->mark.column == 0 && IS_BOM(parser->buffer))
- SKIP(parser);
+ SKIPN(parser,3); /* UTF-8 BOM is 3 bytes */
/*
* Eat whitespaces.
@@ -2681,31 +2694,6 @@
octet = (AS_HEX_AT(parser->buffer, 1) << 4) + AS_HEX_AT(parser->buffer, 2);
- /* If it is the leading octet, determine the length of the UTF-8 sequence. */
-
- if (!width)
- {
- width = (octet & 0x80) == 0x00 ? 1 :
- (octet & 0xE0) == 0xC0 ? 2 :
- (octet & 0xF0) == 0xE0 ? 3 :
- (octet & 0xF8) == 0xF0 ? 4 : 0;
- if (!width) {
- return yaml_parser_set_scanner_error(parser, directive ?
- "while parsing a %TAG directive" : "while parsing a tag",
- start_mark, "found an incorrect leading UTF-8 octet");
- }
- }
- else
- {
- /* Check if the trailing octet is correct. */
-
- if ((octet & 0xC0) != 0x80) {
- return yaml_parser_set_scanner_error(parser, directive ?
- "while parsing a %TAG directive" : "while parsing a tag",
- start_mark, "found an incorrect trailing UTF-8 octet");
- }
- }
-
/* Copy the octet and move the pointers. */
*(string->pointer++) = octet;
@@ -3102,6 +3090,7 @@
else if (!single && CHECK(parser->buffer, '\\'))
{
size_t code_length = 0;
+ int raw_code = 0;
if (!STRING_EXTEND(parser, string)) goto error;
@@ -3188,6 +3177,11 @@
code_length = 2;
break;
+ case 'X':
+ code_length = 2;
+ raw_code = 1;
+ break;
+
case 'u':
code_length = 4;
break;
@@ -3233,7 +3227,7 @@
goto error;
}
- if (value <= 0x7F) {
+ if (value <= 0x7F || raw_code) {
*(string.pointer++) = value;
}
else if (value <= 0x7FF) {
@@ -3262,7 +3256,7 @@
else
{
- /* It is a non-escaped non-blank character. */
+ /* It is a non-escaped non-blank byte. */
if (!READ(parser, string)) goto error;
}
@@ -3474,7 +3468,7 @@
}
}
- /* Copy the character. */
+ /* Copy the byte. */
if (!READ(parser, string)) goto error;
diff -ur original_src/yaml_private.h src/yaml_private.h
--- original_src/yaml_private.h 2008-12-27 15:18:21.000000000 -0800
+++ src/yaml_private.h 2009-08-18 17:57:05.000000000 -0700
@@ -25,7 +25,7 @@
yaml_strdup(const yaml_char_t *);
/*
- * Reader: Ensure that the buffer contains at least `length` characters.
+ * Reader: Ensure that the buffer contains at least `length` bytes.
*/
YAML_DECLARE(int)
@@ -237,6 +237,7 @@
/*
* Check if the character can be printed unescaped.
+ * Only correct if you know you are looking at valid UTF-8!
*/
#define IS_PRINTABLE_AT(string,offset) \
@@ -355,6 +356,7 @@
/*
* Determine the width of the character.
+ * Only correct if you know you are looking at valid UTF-8!
*/
#define WIDTH_AT(string,offset) \
@@ -366,30 +368,23 @@
#define WIDTH(string) WIDTH_AT((string),0)
/*
- * Move the string pointer to the next character.
+ * Move the string pointer to the next byte.
*/
-#define MOVE(string) ((string).pointer += WIDTH((string)))
+#define MOVE(string) ((string).pointer++)
+
+#define MOVEN(string,n) ((string).pointer += n)
/*
- * Copy a character and move the pointers of both strings.
+ * Copy a byte and move the pointers of both strings.
*/
#define COPY(string_a,string_b) \
- ((*(string_b).pointer & 0x80) == 0x00 ? \
- (*((string_a).pointer++) = *((string_b).pointer++)) : \
- (*(string_b).pointer & 0xE0) == 0xC0 ? \
- (*((string_a).pointer++) = *((string_b).pointer++), \
- *((string_a).pointer++) = *((string_b).pointer++)) : \
- (*(string_b).pointer & 0xF0) == 0xE0 ? \
- (*((string_a).pointer++) = *((string_b).pointer++), \
- *((string_a).pointer++) = *((string_b).pointer++), \
- *((string_a).pointer++) = *((string_b).pointer++)) : \
- (*(string_b).pointer & 0xF8) == 0xF0 ? \
- (*((string_a).pointer++) = *((string_b).pointer++), \
- *((string_a).pointer++) = *((string_b).pointer++), \
- *((string_a).pointer++) = *((string_b).pointer++), \
- *((string_a).pointer++) = *((string_b).pointer++)) : 0)
+ (*((string_a).pointer++) = *((string_b).pointer++))
+
+#define COPYN(string_a,string_b,n) \
+ (memcpy(string_a.pointer, string_b.pointer, n), \
+ string_a.pointer += n, string_b.pointer += n)
/*
* Stack and queue management.