From: Bruno Haible Date: Tue, 28 Dec 2021 23:06:17 +0000 (+0100) Subject: unilbrk: Update handling of regional indicators for Unicode 10.0.0. X-Git-Tag: v1.0~2459 X-Git-Url: https://gitweb.git.savannah.gnu.org/gitweb/?a=commitdiff_plain;h=6967c03d3b03a93c050354e05b97b64360aa9c61;p=gnulib.git unilbrk: Update handling of regional indicators for Unicode 10.0.0. * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Add code for handling regional indicators. * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks): Likewise. * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks): Likewise. * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test regarding regional indicators. * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. --- diff --git a/ChangeLog b/ChangeLog index 1ae731cf10..74a988abd2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +2021-12-28 Bruno Haible + + unilbrk: Update handling of regional indicators for Unicode 10.0.0. + * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Add + code for handling regional indicators. + * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks): + Likewise. + * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks): + Likewise. + * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test + regarding regional indicators. + * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. + 2021-12-28 Bruno Haible unilbrk: Update handling of zero-width joiner for Unicode 10.0.0. diff --git a/lib/unilbrk/u16-possible-linebreaks.c b/lib/unilbrk/u16-possible-linebreaks.c index e0f0ff06b7..7ad88367c7 100644 --- a/lib/unilbrk/u16-possible-linebreaks.c +++ b/lib/unilbrk/u16-possible-linebreaks.c @@ -49,6 +49,10 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char /* Don't break inside multibyte characters. */ memset (p, UC_BREAK_PROHIBITED, n); + /* Number of consecutive regional indicator (RI) characters seen + immediately before the current point. */ + size_t ri_count = 0; + do { ucs4_t uc; @@ -152,6 +156,15 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char /* (LB8a) Don't break right after a zero-width joiner. */ *p = UC_BREAK_PROHIBITED; } + else if (last_prop == LBP_RI && prop == LBP_RI) + { + /* (LB30a) Break between two regional indicator symbols + if and only if there are an even number of regional + indicators preceding the position of the break. */ + *p = (seen_space != NULL || (ri_count % 2) == 0 + ? UC_BREAK_POSSIBLE + : UC_BREAK_PROHIBITED); + } else { switch (unilbrk_table [last_prop] [prop]) @@ -176,6 +189,11 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char prev_prop = prop; } + if (prop == LBP_RI) + ri_count++; + else + ri_count = 0; + s += count; p += count; } diff --git a/lib/unilbrk/u32-possible-linebreaks.c b/lib/unilbrk/u32-possible-linebreaks.c index ab973e911c..0c84ff4f1a 100644 --- a/lib/unilbrk/u32-possible-linebreaks.c +++ b/lib/unilbrk/u32-possible-linebreaks.c @@ -44,6 +44,10 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char int last_prop = LBP_BK; /* line break property of last non-space character */ char *seen_space = NULL; /* Was a space seen after the last non-space character? */ + /* Number of consecutive regional indicator (RI) characters seen + immediately before the current point. */ + size_t ri_count = 0; + do { ucs4_t uc = *s; @@ -146,6 +150,15 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char /* (LB8a) Don't break right after a zero-width joiner. */ *p = UC_BREAK_PROHIBITED; } + else if (last_prop == LBP_RI && prop == LBP_RI) + { + /* (LB30a) Break between two regional indicator symbols + if and only if there are an even number of regional + indicators preceding the position of the break. */ + *p = (seen_space != NULL || (ri_count % 2) == 0 + ? UC_BREAK_POSSIBLE + : UC_BREAK_PROHIBITED); + } else { switch (unilbrk_table [last_prop] [prop]) @@ -170,6 +183,11 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char prev_prop = prop; } + if (prop == LBP_RI) + ri_count++; + else + ri_count = 0; + s++; p++; } diff --git a/lib/unilbrk/u8-possible-linebreaks.c b/lib/unilbrk/u8-possible-linebreaks.c index 86f1ce8f13..b631ddeec6 100644 --- a/lib/unilbrk/u8-possible-linebreaks.c +++ b/lib/unilbrk/u8-possible-linebreaks.c @@ -49,6 +49,10 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * /* Don't break inside multibyte characters. */ memset (p, UC_BREAK_PROHIBITED, n); + /* Number of consecutive regional indicator (RI) characters seen + immediately before the current point. */ + size_t ri_count = 0; + do { ucs4_t uc; @@ -152,6 +156,15 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * /* (LB8a) Don't break right after a zero-width joiner. */ *p = UC_BREAK_PROHIBITED; } + else if (last_prop == LBP_RI && prop == LBP_RI) + { + /* (LB30a) Break between two regional indicator symbols + if and only if there are an even number of regional + indicators preceding the position of the break. */ + *p = (seen_space != NULL || (ri_count % 2) == 0 + ? UC_BREAK_POSSIBLE + : UC_BREAK_PROHIBITED); + } else { switch (unilbrk_table [last_prop] [prop]) @@ -176,6 +189,11 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * prev_prop = prop; } + if (prop == LBP_RI) + ri_count++; + else + ri_count = 0; + s += count; p += count; } diff --git a/tests/unilbrk/test-u16-possible-linebreaks.c b/tests/unilbrk/test-u16-possible-linebreaks.c index d8dd863472..87d712fcc6 100644 --- a/tests/unilbrk/test-u16-possible-linebreaks.c +++ b/tests/unilbrk/test-u16-possible-linebreaks.c @@ -164,5 +164,20 @@ main () free (p); } + /* Test line breaking of regional indicators. */ + { + static const uint16_t input[8] = + { 0xD83C, 0xDDE9, 0xD83C, 0xDDEA, 0xD83C, 0xDDEB, 0xD83C, 0xDDF7 }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 8; i++) + { + ASSERT (p[i] == (i == 4 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u32-possible-linebreaks.c b/tests/unilbrk/test-u32-possible-linebreaks.c index 97c08ad5a9..c140ca3b55 100644 --- a/tests/unilbrk/test-u32-possible-linebreaks.c +++ b/tests/unilbrk/test-u32-possible-linebreaks.c @@ -164,5 +164,20 @@ main () free (p); } + /* Test line breaking of regional indicators. */ + { + static const uint32_t input[4] = + { 0x1F1E9, 0x1F1EA, 0x1F1EB, 0x1F1F7 }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 4; i++) + { + ASSERT (p[i] == (i == 2 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u8-possible-linebreaks.c b/tests/unilbrk/test-u8-possible-linebreaks.c index 9dd5666954..0fdb66d367 100644 --- a/tests/unilbrk/test-u8-possible-linebreaks.c +++ b/tests/unilbrk/test-u8-possible-linebreaks.c @@ -149,5 +149,20 @@ main () free (p); } + /* Test line breaking of regional indicators. */ + { + static const uint8_t input[16] = + "\360\237\207\251\360\237\207\252\360\237\207\253\360\237\207\267"; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 16; i++) + { + ASSERT (p[i] == (i == 8 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; }