From 2995fb5e993a5d7434d96465758087b35a1488ac Mon Sep 17 00:00:00 2001 From: Bruno Haible Date: Wed, 29 Dec 2021 00:06:11 +0100 Subject: [PATCH] unilbrk: Update handling of zero-width joiner for Unicode 10.0.0. * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Update code for zero-width joiner handling to match UAX #14 for Unicode 10.0.0. * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks): Likewise. * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks): Likewise. * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test regarding zero-width joiner. * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. --- ChangeLog | 14 ++++++++++ lib/unilbrk/u16-possible-linebreaks.c | 22 +++++++++++---- lib/unilbrk/u32-possible-linebreaks.c | 22 +++++++++++---- lib/unilbrk/u8-possible-linebreaks.c | 22 +++++++++++---- tests/unilbrk/test-u16-possible-linebreaks.c | 29 ++++++++++++++++++++ tests/unilbrk/test-u32-possible-linebreaks.c | 29 ++++++++++++++++++++ tests/unilbrk/test-u8-possible-linebreaks.c | 27 ++++++++++++++++++ 7 files changed, 147 insertions(+), 18 deletions(-) diff --git a/ChangeLog b/ChangeLog index f0a216b6b9..1ae731cf10 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +2021-12-28 Bruno Haible + + unilbrk: Update handling of zero-width joiner for Unicode 10.0.0. + * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Update + code for zero-width joiner handling to match UAX #14 for Unicode 10.0.0. + * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks): + Likewise. + * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks): + Likewise. + * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test + regarding zero-width joiner. + * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. + 2021-12-28 Bruno Haible unilbrk: Update handling of combining marks for Unicode 10.0.0. diff --git a/lib/unilbrk/u16-possible-linebreaks.c b/lib/unilbrk/u16-possible-linebreaks.c index 4c01720096..e0f0ff06b7 100644 --- a/lib/unilbrk/u16-possible-linebreaks.c +++ b/lib/unilbrk/u16-possible-linebreaks.c @@ -42,6 +42,7 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char { int LBP_AI_REPLACEMENT = (is_cjk_encoding (encoding) ? LBP_ID : LBP_AL); const uint16_t *s_end = s + n; + int prev_prop = LBP_BK; /* line break property of last character */ int last_prop = LBP_BK; /* line break property of last non-space character */ char *seen_space = NULL; /* Was a space seen after the last non-space character? */ @@ -58,6 +59,7 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char { /* (LB4,LB5,LB6) Mandatory break. */ *p = UC_BREAK_MANDATORY; + prev_prop = LBP_BK; last_prop = LBP_BK; seen_space = NULL; } @@ -97,16 +99,16 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char last_prop = LBP_ZW; seen_space = NULL; } - else if (prop == LBP_CM) + else if (prop == LBP_CM || prop == LBP_ZWJ) { - /* (LB9) Don't break just before a combining character, except - immediately after a mandatory break character, space, or - zero-width space. */ + /* (LB9) Don't break just before a combining character or + zero-width joiner, except immediately after a mandatory + break character, space, or zero-width space. */ if (last_prop == LBP_BK) { /* (LB4,LB5,LB6) Don't break at the beginning of a line. */ *p = UC_BREAK_PROHIBITED; - /* Treat CM as AL. */ + /* (LB10) Treat CM or ZWJ as AL. */ last_prop = LBP_AL; seen_space = NULL; } @@ -118,7 +120,7 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char character as base for combining marks" because now the NBSP CM sequence is recommended instead of SP CM. */ *p = UC_BREAK_POSSIBLE; - /* Treat CM as AL. */ + /* (LB10) Treat CM or ZWJ as AL. */ last_prop = LBP_AL; seen_space = NULL; } @@ -144,6 +146,12 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char /* (LB8) Break after zero-width space. */ *p = UC_BREAK_POSSIBLE; } + else if (prev_prop == LBP_ZWJ + && (prop == LBP_ID || prop == LBP_EB || prop == LBP_EM)) + { + /* (LB8a) Don't break right after a zero-width joiner. */ + *p = UC_BREAK_PROHIBITED; + } else { switch (unilbrk_table [last_prop] [prop]) @@ -164,6 +172,8 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char last_prop = prop; seen_space = NULL; } + + prev_prop = prop; } s += count; diff --git a/lib/unilbrk/u32-possible-linebreaks.c b/lib/unilbrk/u32-possible-linebreaks.c index 562419ad96..ab973e911c 100644 --- a/lib/unilbrk/u32-possible-linebreaks.c +++ b/lib/unilbrk/u32-possible-linebreaks.c @@ -40,6 +40,7 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char { int LBP_AI_REPLACEMENT = (is_cjk_encoding (encoding) ? LBP_ID : LBP_AL); const uint32_t *s_end = s + n; + int prev_prop = LBP_BK; /* line break property of last character */ int last_prop = LBP_BK; /* line break property of last non-space character */ char *seen_space = NULL; /* Was a space seen after the last non-space character? */ @@ -52,6 +53,7 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char { /* (LB4,LB5,LB6) Mandatory break. */ *p = UC_BREAK_MANDATORY; + prev_prop = LBP_BK; last_prop = LBP_BK; seen_space = NULL; } @@ -91,16 +93,16 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char last_prop = LBP_ZW; seen_space = NULL; } - else if (prop == LBP_CM) + else if (prop == LBP_CM || prop == LBP_ZWJ) { - /* (LB9) Don't break just before a combining character, except - immediately after a mandatory break character, space, or - zero-width space. */ + /* (LB9) Don't break just before a combining character or + zero-width joiner, except immediately after a mandatory + break character, space, or zero-width space. */ if (last_prop == LBP_BK) { /* (LB4,LB5,LB6) Don't break at the beginning of a line. */ *p = UC_BREAK_PROHIBITED; - /* Treat CM as AL. */ + /* (LB10) Treat CM or ZWJ as AL. */ last_prop = LBP_AL; seen_space = NULL; } @@ -112,7 +114,7 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char character as base for combining marks" because now the NBSP CM sequence is recommended instead of SP CM. */ *p = UC_BREAK_POSSIBLE; - /* Treat CM as AL. */ + /* (LB10) Treat CM or ZWJ as AL. */ last_prop = LBP_AL; seen_space = NULL; } @@ -138,6 +140,12 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char /* (LB8) Break after zero-width space. */ *p = UC_BREAK_POSSIBLE; } + else if (prev_prop == LBP_ZWJ + && (prop == LBP_ID || prop == LBP_EB || prop == LBP_EM)) + { + /* (LB8a) Don't break right after a zero-width joiner. */ + *p = UC_BREAK_PROHIBITED; + } else { switch (unilbrk_table [last_prop] [prop]) @@ -158,6 +166,8 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char last_prop = prop; seen_space = NULL; } + + prev_prop = prop; } s++; diff --git a/lib/unilbrk/u8-possible-linebreaks.c b/lib/unilbrk/u8-possible-linebreaks.c index d7ab680203..86f1ce8f13 100644 --- a/lib/unilbrk/u8-possible-linebreaks.c +++ b/lib/unilbrk/u8-possible-linebreaks.c @@ -42,6 +42,7 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * { int LBP_AI_REPLACEMENT = (is_cjk_encoding (encoding) ? LBP_ID : LBP_AL); const uint8_t *s_end = s + n; + int prev_prop = LBP_BK; /* line break property of last character */ int last_prop = LBP_BK; /* line break property of last non-space character */ char *seen_space = NULL; /* Was a space seen after the last non-space character? */ @@ -58,6 +59,7 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * { /* (LB4,LB5,LB6) Mandatory break. */ *p = UC_BREAK_MANDATORY; + prev_prop = LBP_BK; last_prop = LBP_BK; seen_space = NULL; } @@ -97,16 +99,16 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * last_prop = LBP_ZW; seen_space = NULL; } - else if (prop == LBP_CM) + else if (prop == LBP_CM || prop == LBP_ZWJ) { - /* (LB9) Don't break just before a combining character, except - immediately after a mandatory break character, space, or - zero-width space. */ + /* (LB9) Don't break just before a combining character or + zero-width joiner, except immediately after a mandatory + break character, space, or zero-width space. */ if (last_prop == LBP_BK) { /* (LB4,LB5,LB6) Don't break at the beginning of a line. */ *p = UC_BREAK_PROHIBITED; - /* Treat CM as AL. */ + /* (LB10) Treat CM or ZWJ as AL. */ last_prop = LBP_AL; seen_space = NULL; } @@ -118,7 +120,7 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * character as base for combining marks" because now the NBSP CM sequence is recommended instead of SP CM. */ *p = UC_BREAK_POSSIBLE; - /* Treat CM as AL. */ + /* (LB10) Treat CM or ZWJ as AL. */ last_prop = LBP_AL; seen_space = NULL; } @@ -144,6 +146,12 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * /* (LB8) Break after zero-width space. */ *p = UC_BREAK_POSSIBLE; } + else if (prev_prop == LBP_ZWJ + && (prop == LBP_ID || prop == LBP_EB || prop == LBP_EM)) + { + /* (LB8a) Don't break right after a zero-width joiner. */ + *p = UC_BREAK_PROHIBITED; + } else { switch (unilbrk_table [last_prop] [prop]) @@ -164,6 +172,8 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * last_prop = prop; seen_space = NULL; } + + prev_prop = prop; } s += count; diff --git a/tests/unilbrk/test-u16-possible-linebreaks.c b/tests/unilbrk/test-u16-possible-linebreaks.c index 2d0aef0142..d8dd863472 100644 --- a/tests/unilbrk/test-u16-possible-linebreaks.c +++ b/tests/unilbrk/test-u16-possible-linebreaks.c @@ -135,5 +135,34 @@ main () free (p); } + /* Test line breaking of zero-width joiners (U+200D). */ + { + static const uint16_t input[39] = + { + 0x6709, 0x7121, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x6578, 0x3002, '\n', + 0x6709, 0x200D, 0x7121, 0x200D, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x200D, 0x6578, 0x3002, '\n', + 0x4F60, 0x2014, 0x4E0D, '\n', + 0x4F60, 0x2014, 0x200D, 0x4E0D, '\n', + 0x261D, 0xD83C, 0xDFFF, '\n', + 0x261D, 0x200D, 0xD83C, 0xDFFF, '\n', + }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 39; i++) + { + ASSERT (p[i] == (i == 8 || i == 20 + || i == 24 || i == 29 + || i == 33 || i == 38 ? UC_BREAK_MANDATORY : + i == 1 || i == 2 || i == 3 || i == 4 || i == 5 || i == 6 + || i == 14 || i == 15 || i == 16 + || i == 22 || i == 23 + || i == 26 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u32-possible-linebreaks.c b/tests/unilbrk/test-u32-possible-linebreaks.c index ab5840d589..97c08ad5a9 100644 --- a/tests/unilbrk/test-u32-possible-linebreaks.c +++ b/tests/unilbrk/test-u32-possible-linebreaks.c @@ -135,5 +135,34 @@ main () free (p); } + /* Test line breaking of zero-width joiners (U+200D). */ + { + static const uint32_t input[37] = + { + 0x6709, 0x7121, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x6578, 0x3002, '\n', + 0x6709, 0x200D, 0x7121, 0x200D, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x200D, 0x6578, 0x3002, '\n', + 0x4F60, 0x2014, 0x4E0D, '\n', + 0x4F60, 0x2014, 0x200D, 0x4E0D, '\n', + 0x261D, 0x1F3FF, '\n', + 0x261D, 0x200D, 0x1F3FF, '\n', + }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 37; i++) + { + ASSERT (p[i] == (i == 8 || i == 20 + || i == 24 || i == 29 + || i == 32 || i == 36 ? UC_BREAK_MANDATORY : + i == 1 || i == 2 || i == 3 || i == 4 || i == 5 || i == 6 + || i == 14 || i == 15 || i == 16 + || i == 22 || i == 23 + || i == 26 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u8-possible-linebreaks.c b/tests/unilbrk/test-u8-possible-linebreaks.c index d4625adca0..9dd5666954 100644 --- a/tests/unilbrk/test-u8-possible-linebreaks.c +++ b/tests/unilbrk/test-u8-possible-linebreaks.c @@ -122,5 +122,32 @@ main () free (p); } + /* Test line breaking of zero-width joiners (U+200D). */ + { + static const uint8_t input[101] = + "\346\234\211\347\204\241\347\252\256\345\244\232\345\200\213\347\264\240\346\225\270\343\200\202\n" /* "有無窮多個素數。" */ + "\346\234\211\342\200\215\347\204\241\342\200\215\347\252\256\345\244\232\345\200\213\347\264\240\342\200\215\346\225\270\343\200\202\n" + "\344\275\240\342\200\224\344\270\215\n" /* "你—不" */ + "\344\275\240\342\200\224\342\200\215\344\270\215\n" + "\342\230\235\360\237\217\277\n" /* "☝🏿" */ + "\342\230\235\342\200\215\360\237\217\277\n"; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 101; i++) + { + ASSERT (p[i] == (i == 24 || i == 58 + || i == 68 || i == 81 + || i == 89 || i == 100 ? UC_BREAK_MANDATORY : + i == 3 || i == 6 || i == 9 || i == 12 || i == 15 || i == 18 + || i == 40 || i == 43 || i == 46 + || i == 62 || i == 65 + || i == 72 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } -- 2.39.5