+2023-07-25 Bruno Haible <bruno@clisp.org>
+
+ unistr/u8-*: Make Unicode decoder more Unicode Standard compliant.
+ Based on a remark by Paul Eggert in
+ <https://lists.gnu.org/archive/html/bug-gnulib/2023-07/msg00120.html>.
+ * tests/unistr/test-u8-mbtouc.c (test_safe_function): Change expected
+ results for "non-shortest form" or out-of-range byte sequences. Add new
+ test cases of incomplete well-formed byte sequences.
+ * tests/unistr/test-u8-mbsnlen.c (main): Likewise.
+ * lib/unistr/u8-mbtouc-aux.c (u8_mbtouc_aux): Reject a first byte in the
+ range 0xF5..0xF7 as invalid. Distinguish incomplete from invalid byte
+ sequences correctly. For the former, return only the number of bytes in
+ the maximal well-formed subpart.
+ * lib/unistr/u8-mbtouc.c (u8_mbtouc): Likewise.
+ * lib/unistr/u8-check.c (u8_check): Reject a first byte in the range
+ 0xF5..0xF7 as invalid.
+ * lib/unistr/u8-mblen.c (u8_mblen): Likewise.
+ * lib/unistr/u8-mbtoucr.c (u8_mbtoucr): Likewise.
+ * lib/unistr/u8-strmbtouc.c (u8_strmbtouc): Likewise.
+ * lib/unistr/u8-strmblen.c (u8_strmblen): Likewise.
+ * lib/unistr/u8-prev.c (u8_prev): Likewise.
+
2023-07-24 Bruno Haible <bruno@clisp.org>
fnmatch: Overcome wchar_t limitations.
continue;
}
}
- else if (c < 0xf8)
+ else if (c <= 0xf4)
{
if (s + 4 <= s_end
&& (s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
&& (s[3] ^ 0x80) < 0x40
&& (c >= 0xf1 || s[1] >= 0x90)
- && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90)))
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
{
s += 4;
continue;
&& (c != 0xed || s[1] < 0xa0))
return 3;
}
- else if (c < 0xf8)
+ else if (c <= 0xf4)
{
if (n >= 4
&& (s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
&& (s[3] ^ 0x80) < 0x40
&& (c >= 0xf1 || s[1] >= 0x90)
- && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90)))
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
return 4;
}
}
{
if (n >= 3)
{
- if ((s[1] ^ 0x80) < 0x40)
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xe1 || s[1] >= 0xa0)
+ && (c != 0xed || s[1] < 0xa0))
{
if ((s[2] ^ 0x80) < 0x40)
{
- if ((c >= 0xe1 || s[1] >= 0xa0)
- && (c != 0xed || s[1] < 0xa0))
- {
- *puc = ((unsigned int) (c & 0x0f) << 12)
- | ((unsigned int) (s[1] ^ 0x80) << 6)
- | (unsigned int) (s[2] ^ 0x80);
- return 3;
- }
- /* invalid multibyte character */
- *puc = 0xfffd;
+ *puc = ((unsigned int) (c & 0x0f) << 12)
+ | ((unsigned int) (s[1] ^ 0x80) << 6)
+ | (unsigned int) (s[2] ^ 0x80);
return 3;
}
/* invalid multibyte character */
return 2;
}
/* invalid multibyte character */
+ *puc = 0xfffd;
+ return 1;
}
else
{
- /* incomplete multibyte character */
*puc = 0xfffd;
- if (n == 1 || (s[1] ^ 0x80) >= 0x40)
- return 1;
+ if (n == 1)
+ {
+ /* incomplete multibyte character */
+ return 1;
+ }
else
- return 2;
+ {
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xe1 || s[1] >= 0xa0)
+ && (c != 0xed || s[1] < 0xa0))
+ {
+ /* incomplete multibyte character */
+ return 2;
+ }
+ else
+ {
+ /* invalid multibyte character */
+ return 1;
+ }
+ }
}
}
- else if (c < 0xf8)
+ else if (c <= 0xf4)
{
if (n >= 4)
{
- if ((s[1] ^ 0x80) < 0x40)
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xf1 || s[1] >= 0x90)
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
{
if ((s[2] ^ 0x80) < 0x40)
{
if ((s[3] ^ 0x80) < 0x40)
{
- if ((c >= 0xf1 || s[1] >= 0x90)
- && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90)))
- {
- *puc = ((unsigned int) (c & 0x07) << 18)
- | ((unsigned int) (s[1] ^ 0x80) << 12)
- | ((unsigned int) (s[2] ^ 0x80) << 6)
- | (unsigned int) (s[3] ^ 0x80);
- return 4;
- }
- /* invalid multibyte character */
- *puc = 0xfffd;
+ *puc = ((unsigned int) (c & 0x07) << 18)
+ | ((unsigned int) (s[1] ^ 0x80) << 12)
+ | ((unsigned int) (s[2] ^ 0x80) << 6)
+ | (unsigned int) (s[3] ^ 0x80);
return 4;
}
/* invalid multibyte character */
return 2;
}
/* invalid multibyte character */
+ *puc = 0xfffd;
+ return 1;
}
else
{
- /* incomplete multibyte character */
*puc = 0xfffd;
- if (n == 1 || (s[1] ^ 0x80) >= 0x40)
- return 1;
- else if (n == 2 || (s[2] ^ 0x80) >= 0x40)
- return 2;
+ if (n == 1)
+ {
+ /* incomplete multibyte character */
+ return 1;
+ }
else
- return 3;
+ {
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xf1 || s[1] >= 0x90)
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
+ {
+ if (n == 2)
+ {
+ /* incomplete multibyte character */
+ return 2;
+ }
+ else
+ {
+ if ((s[2] ^ 0x80) < 0x40)
+ {
+ /* incomplete multibyte character */
+ return 3;
+ }
+ else
+ {
+ /* invalid multibyte character */
+ return 2;
+ }
+ }
+ }
+ else
+ {
+ /* invalid multibyte character */
+ return 1;
+ }
+ }
}
}
}
{
if (n >= 3)
{
- if ((s[1] ^ 0x80) < 0x40)
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xe1 || s[1] >= 0xa0)
+ && (c != 0xed || s[1] < 0xa0))
{
if ((s[2] ^ 0x80) < 0x40)
{
- if ((c >= 0xe1 || s[1] >= 0xa0)
- && (c != 0xed || s[1] < 0xa0))
- {
- *puc = ((unsigned int) (c & 0x0f) << 12)
- | ((unsigned int) (s[1] ^ 0x80) << 6)
- | (unsigned int) (s[2] ^ 0x80);
- return 3;
- }
- /* invalid multibyte character */
- *puc = 0xfffd;
+ *puc = ((unsigned int) (c & 0x0f) << 12)
+ | ((unsigned int) (s[1] ^ 0x80) << 6)
+ | (unsigned int) (s[2] ^ 0x80);
return 3;
}
/* invalid multibyte character */
return 2;
}
/* invalid multibyte character */
+ *puc = 0xfffd;
+ return 1;
}
else
{
- /* incomplete multibyte character */
*puc = 0xfffd;
- if (n == 1 || (s[1] ^ 0x80) >= 0x40)
- return 1;
+ if (n == 1)
+ {
+ /* incomplete multibyte character */
+ return 1;
+ }
else
- return 2;
+ {
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xe1 || s[1] >= 0xa0)
+ && (c != 0xed || s[1] < 0xa0))
+ {
+ /* incomplete multibyte character */
+ return 2;
+ }
+ else
+ {
+ /* invalid multibyte character */
+ return 1;
+ }
+ }
}
}
- else if (c < 0xf8)
+ else if (c <= 0xf4)
{
if (n >= 4)
{
- if ((s[1] ^ 0x80) < 0x40)
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xf1 || s[1] >= 0x90)
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
{
if ((s[2] ^ 0x80) < 0x40)
{
if ((s[3] ^ 0x80) < 0x40)
{
- if ((c >= 0xf1 || s[1] >= 0x90)
- && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90)))
- {
- *puc = ((unsigned int) (c & 0x07) << 18)
- | ((unsigned int) (s[1] ^ 0x80) << 12)
- | ((unsigned int) (s[2] ^ 0x80) << 6)
- | (unsigned int) (s[3] ^ 0x80);
- return 4;
- }
- /* invalid multibyte character */
- *puc = 0xfffd;
+ *puc = ((unsigned int) (c & 0x07) << 18)
+ | ((unsigned int) (s[1] ^ 0x80) << 12)
+ | ((unsigned int) (s[2] ^ 0x80) << 6)
+ | (unsigned int) (s[3] ^ 0x80);
return 4;
}
/* invalid multibyte character */
return 2;
}
/* invalid multibyte character */
+ *puc = 0xfffd;
+ return 1;
}
else
{
- /* incomplete multibyte character */
*puc = 0xfffd;
- if (n == 1 || (s[1] ^ 0x80) >= 0x40)
- return 1;
- else if (n == 2 || (s[2] ^ 0x80) >= 0x40)
- return 2;
+ if (n == 1)
+ {
+ /* incomplete multibyte character */
+ return 1;
+ }
else
- return 3;
+ {
+ if ((s[1] ^ 0x80) < 0x40
+ && (c >= 0xf1 || s[1] >= 0x90)
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
+ {
+ if (n == 2)
+ {
+ /* incomplete multibyte character */
+ return 2;
+ }
+ else
+ {
+ if ((s[2] ^ 0x80) < 0x40)
+ {
+ /* incomplete multibyte character */
+ return 3;
+ }
+ else
+ {
+ /* invalid multibyte character */
+ return 2;
+ }
+ }
+ }
+ else
+ {
+ /* invalid multibyte character */
+ return 1;
+ }
+ }
}
}
}
return -2;
}
}
- else if (c < 0xf8)
+ else if (c <= 0xf4)
{
if (n >= 2)
{
if ((s[1] ^ 0x80) < 0x40
&& (c >= 0xf1 || s[1] >= 0x90)
- && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90)))
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
{
if (n >= 3)
{
{
uint8_t c_4 = s[-4];
- if (c_4 >= 0xf0 && c_4 < 0xf8
+ if (c_4 >= 0xf0 && c_4 <= 0xf4
&& (c_4 >= 0xf1 || c_3 >= 0x90)
- && (c_4 < 0xf4 || (c_4 == 0xf4 && c_3 < 0x90)))
+ && (c_4 < 0xf4 || (/* c_4 == 0xf4 && */ c_3 < 0x90)))
{
*puc = ((unsigned int) (c_4 & 0x07) << 18)
| ((unsigned int) (c_3 ^ 0x80) << 12)
&& (c != 0xed || s[1] < 0xa0))
return 3;
}
- else if (c < 0xf8)
+ else if (c <= 0xf4)
{
if ((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
&& (s[3] ^ 0x80) < 0x40
&& (c >= 0xf1 || s[1] >= 0x90)
- && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90)))
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
return 4;
}
}
return 3;
}
}
- else if (c < 0xf8)
+ else if (c <= 0xf4)
{
if ((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
&& (s[3] ^ 0x80) < 0x40
&& (c >= 0xf1 || s[1] >= 0x90)
- && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90)))
+ && (c < 0xf4 || (/* c == 0xf4 && */ s[1] < 0x90)))
{
*puc = ((unsigned int) (c & 0x07) << 18)
| ((unsigned int) (s[1] ^ 0x80) << 12)
that a "malformed sequence" is interpreted in the same way as
"a character that is outside the adopted subset".
Reference:
+ ISO 10646-1 amendment 2
+ <https://www.cl.cam.ac.uk/~mgk25/ucs/ISO-10646-UTF-8.html>
Markus Kuhn: UTF-8 decoder capability and stress test
<https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt>
<https://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html>
+ These old specifications (from ca. 2000) were a bit ambiguous, and the
+ definition of UTF-8 has changed a bit as well. The newer specification
+ we obey is the Unicode Standard, version 15.
+ Reference:
+ Unicode Standard 15.0.0, section 3.9
+ <https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf>
+ pages 124..129, especially table 3-7.
*/
/* 3.1. Test that each unexpected continuation byte is signalled as a
malformed sequence of its own. */
}
/* 3.3.2. 3-byte sequence with last byte missing. */
{
- static const uint8_t input[] = { '"', 0xE0, 0x80, '"' };
+ static const uint8_t input[] = { '"', 0xE0, 0xA0, '"' };
ASSERT (u8_mbsnlen (input, 4) == 3);
}
+ {
+ /* Outdated example: 0xE0 0x80 is an ill-formed sequence. */
+ static const uint8_t input[] = { '"', 0xE0, 0x80, '"' };
+ ASSERT (u8_mbsnlen (input, 4) == 4);
+ }
/* 3.3.7. 3-byte sequence with last byte missing. */
{
static const uint8_t input[] = { '"', 0xEF, 0xBF, '"' };
}
/* 3.3.3. 4-byte sequence with last byte missing. */
{
- static const uint8_t input[] = { '"', 0xF0, 0x80, 0x80, '"' };
+ static const uint8_t input[] = { '"', 0xF0, 0x90, 0x80, '"' };
ASSERT (u8_mbsnlen (input, 5) == 3);
}
+ {
+ /* Outdated example: 0xF0 0x80 is an ill-formed sequence. */
+ static const uint8_t input[] = { '"', 0xF0, 0x80, 0x80, '"' };
+ ASSERT (u8_mbsnlen (input, 5) == 5);
+ }
/* 3.3.8. 4-byte sequence with last byte missing. */
{
- static const uint8_t input[] = { '"', 0xF7, 0xBF, 0xBF, '"' };
+ static const uint8_t input[] = { '"', 0xF3, 0xBF, 0xBF, '"' };
ASSERT (u8_mbsnlen (input, 5) == 3);
}
+ {
+ /* Outdated example: 0xF7 is an invalid first byte. */
+ static const uint8_t input[] = { '"', 0xF7, 0xBF, 0xBF, '"' };
+ ASSERT (u8_mbsnlen (input, 5) == 5);
+ }
return 0;
}
that a "malformed sequence" is interpreted in the same way as
"a character that is outside the adopted subset".
Reference:
+ ISO 10646-1 amendment 2
+ <https://www.cl.cam.ac.uk/~mgk25/ucs/ISO-10646-UTF-8.html>
Markus Kuhn: UTF-8 decoder capability and stress test
<https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt>
<https://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html>
+ These old specifications (from ca. 2000) were a bit ambiguous, and the
+ definition of UTF-8 has changed a bit as well. The newer specification
+ we obey is the Unicode Standard, version 15.
+ Reference:
+ Unicode Standard 15.0.0, section 3.9
+ <https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf>
+ pages 124..129, especially table 3-7.
*/
/* 3.1. Test that each unexpected continuation byte is signalled as a
malformed sequence of its own. */
}
/* 3.3.2. 3-byte sequence with last byte missing. */
{
- static const uint8_t input[] = { '"', 0xE0, 0x80, '"' };
+ static const uint8_t input[] = { '"', 0xE0, 0xA0, '"' };
uc = 0xBADFACE;
ret = my_u8_mbtouc (&uc, input, 4);
ASSERT (ret == 1);
ASSERT (ret == 1);
ASSERT (uc == 0x0022);
}
+ {
+ /* Outdated example: 0xE0 0x80 is an ill-formed sequence. */
+ static const uint8_t input[] = { '"', 0xE0, 0x80, '"' };
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input, 4);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0x0022);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 1, 3);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 2, 2);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 3, 1);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0x0022);
+ }
/* 3.3.7. 3-byte sequence with last byte missing. */
{
static const uint8_t input[] = { '"', 0xEF, 0xBF, '"' };
}
/* 3.3.3. 4-byte sequence with last byte missing. */
{
- static const uint8_t input[] = { '"', 0xF0, 0x80, 0x80, '"' };
+ static const uint8_t input[] = { '"', 0xF0, 0x90, 0x80, '"' };
uc = 0xBADFACE;
ret = my_u8_mbtouc (&uc, input, 5);
ASSERT (ret == 1);
ASSERT (ret == 1);
ASSERT (uc == 0x0022);
}
+ {
+ /* Outdated example: 0xF0 0x80 is an ill-formed sequence. */
+ static const uint8_t input[] = { '"', 0xF0, 0x80, 0x80, '"' };
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input, 5);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0x0022);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 1, 4);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 2, 3);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 3, 2);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 4, 1);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0x0022);
+ }
/* 3.3.8. 4-byte sequence with last byte missing. */
{
- static const uint8_t input[] = { '"', 0xF7, 0xBF, 0xBF, '"' };
+ static const uint8_t input[] = { '"', 0xF3, 0xBF, 0xBF, '"' };
uc = 0xBADFACE;
ret = my_u8_mbtouc (&uc, input, 5);
ASSERT (ret == 1);
ASSERT (ret == 1);
ASSERT (uc == 0x0022);
}
+ {
+ /* Outdated example: 0xF7 is an invalid first byte. */
+ static const uint8_t input[] = { '"', 0xF7, 0xBF, 0xBF, '"' };
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input, 5);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0x0022);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 1, 4);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 2, 3);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 3, 2);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0xFFFD);
+ uc = 0xBADFACE;
+ ret = my_u8_mbtouc (&uc, input + 4, 1);
+ ASSERT (ret == 1);
+ ASSERT (uc == 0x0022);
+ }
}
int