unilbrk: Fully implement Unicode rule (LB9).
authorBruno Haible <bruno@clisp.org>
Sun, 15 Sep 2024 11:23:04 +0000 (13:23 +0200)
committerBruno Haible <bruno@clisp.org>
Sun, 15 Sep 2024 12:50:47 +0000 (14:50 +0200)
* lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks_loop):
New variable preceding_prop. Don't copy prop to prev_prop and ea to
prev_ea if the conditions of rule (LB9) apply.
* lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks_loop):
Likewise.
* lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks_loop):
Likewise.

ChangeLog
lib/unilbrk/u16-possible-linebreaks.c
lib/unilbrk/u32-possible-linebreaks.c
lib/unilbrk/u8-possible-linebreaks.c

index 7b61ebaf3cb813a60c93cddbeff9478256b091ef..48dca8cbf2d0f78f568e4f1f755624076c6be612 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2024-09-15  Bruno Haible  <bruno@clisp.org>
+
+       unilbrk: Fully implement Unicode rule (LB9).
+       * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks_loop):
+       New variable preceding_prop. Don't copy prop to prev_prop and ea to
+       prev_ea if the conditions of rule (LB9) apply.
+       * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks_loop):
+       Likewise.
+       * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks_loop):
+       Likewise.
+
 2024-09-15  Bruno Haible  <bruno@clisp.org>
 
        unilbrk: Fix bugs in implementation of Unicode rules (LB15a), (LB15b).
index d2d6343234873f8dc3c47fbfdf550ef1ce5fc632..dde1aa606497fe91ed5f428ccaa6ecab0bfca2b3 100644 (file)
@@ -78,17 +78,20 @@ u16_possible_linebreaks_loop (const uint16_t *s, size_t n, const char *encoding,
           lookahead2_prop_ea = PROP_EA (LBP_BK, 0);
         }
 
-      int prev_prop = LBP_BK; /* line break property of last character */
-      int prev_ea = 0;        /* EastAsian property of last character */
-      int prev2_ea = 0;       /* EastAsian property of character before the
-                                 last character */
-      bool prev_initial_hyphen = false; /* the last character was a word-initial
-                                           hyphen or U+2010 */
-      bool prev_nus = false; /* before the last character, there was a character
+      int preceding_prop = LBP_BK; /* line break property of preceding character */
+      int prev_prop = LBP_BK; /* line break property of previous character
+                                 (= last character, ignoring intervening characters of class CM or ZWJ) */
+      int prev_ea = 0;        /* EastAsian property of previous character
+                                 (= last character, ignoring intervening characters of class CM or ZWJ) */
+      int prev2_ea = 0;       /* EastAsian property of character before the previous character */
+      bool prev_initial_hyphen = false; /* the previous character was a
+                                           word-initial hyphen or U+2010 */
+      bool prev_nus = false; /* before the previous character, there was a character
                                 with line break property LBP_NU and since then
                                 only characters with line break property LBP_SY
                                 or LBP_IS */
-      int last_prop = LBP_BK; /* line break property of last non-space character */
+      int last_prop = LBP_BK; /* line break property of last non-space character
+                                 (= last character, ignoring intervening characters of class SP or CM or ZWJ) */
       char *seen_space = NULL; /* Was a space seen after the last non-space character? */
 
       /* Number of consecutive regional indicator (RI) characters seen
@@ -221,7 +224,7 @@ u16_possible_linebreaks_loop (const uint16_t *s, size_t n, const char *encoding,
                       /* (LB8) Break after zero-width space.  */
                       *p = UC_BREAK_POSSIBLE;
                     }
-                  else if (prev_prop == LBP_ZWJ)
+                  else if (preceding_prop == LBP_ZWJ)
                     {
                       /* (LB8a) Don't break right after a zero-width joiner.  */
                       *p = UC_BREAK_PROHIBITED;
@@ -368,21 +371,29 @@ u16_possible_linebreaks_loop (const uint16_t *s, size_t n, const char *encoding,
                   seen_space = NULL;
                 }
 
-              prev_initial_hyphen =
-                (prop == LBP_HY || uc == 0x2010)
-                && (prev_prop == LBP_BK || prev_prop == LBP_CR || prev_prop == LBP_LF
-                    || prev_prop == LBP_SP || prev_prop == LBP_ZW
-                    || prev_prop == LBP_CB || prev_prop == LBP_GL);
-              prev_prop = (prop == LBP_VI && (prev_prop == LBP_AK
-                                              || prev_prop == LBP_AL2
-                                              || prev_prop == LBP_AS)
-                           ? LBP_AKLS_VI :
-                           prev_prop == LBP_HL && (prop == LBP_HY
-                                                   || (prop == LBP_BA && !ea))
-                           ? LBP_HL_BA :
-                           prop);
-              prev2_ea = prev_ea;
-              prev_ea = ea;
+              /* (LB9) Treat X (CM | ZWJ)* as if it were X, where X is any line
+                 break class except BK, CR, LF, NL, SP, or ZW.  */
+              if (!((prop == LBP_CM || prop == LBP_ZWJ)
+                    && !(prev_prop == LBP_BK || prev_prop == LBP_LF || prev_prop == LBP_CR
+                         || prev_prop == LBP_SP || prev_prop == LBP_ZW)))
+                {
+                  prev_initial_hyphen =
+                    (prop == LBP_HY || uc == 0x2010)
+                    && (prev_prop == LBP_BK || prev_prop == LBP_CR || prev_prop == LBP_LF
+                        || prev_prop == LBP_SP || prev_prop == LBP_ZW
+                        || prev_prop == LBP_CB || prev_prop == LBP_GL);
+                  prev_prop = (prop == LBP_VI && (prev_prop == LBP_AK
+                                                  || prev_prop == LBP_AL2
+                                                  || prev_prop == LBP_AS)
+                               ? LBP_AKLS_VI :
+                               prev_prop == LBP_HL && (prop == LBP_HY
+                                                       || (prop == LBP_BA && !ea))
+                               ? LBP_HL_BA :
+                               prop);
+                  prev2_ea = prev_ea;
+                  prev_ea = ea;
+                }
+              preceding_prop = prop;
             }
 
           prev_nus = nus;
index 10f39a8d054f4a93660b59952c7c446374352f39..9cdda2868dad6ac7e543f551032335fa96bdf8e4 100644 (file)
@@ -72,17 +72,20 @@ u32_possible_linebreaks_loop (const uint32_t *s, size_t n, const char *encoding,
           lookahead2_prop_ea = PROP_EA (LBP_BK, 0);
         }
 
-      int prev_prop = LBP_BK; /* line break property of last character */
-      int prev_ea = 0;        /* EastAsian property of last character */
-      int prev2_ea = 0;       /* EastAsian property of character before the
-                                 last character */
-      bool prev_initial_hyphen = false; /* the last character was a word-initial
-                                           hyphen or U+2010 */
-      bool prev_nus = false; /* before the last character, there was a character
+      int preceding_prop = LBP_BK; /* line break property of preceding character */
+      int prev_prop = LBP_BK; /* line break property of previous character
+                                 (= last character, ignoring intervening characters of class CM or ZWJ) */
+      int prev_ea = 0;        /* EastAsian property of previous character
+                                 (= last character, ignoring intervening characters of class CM or ZWJ) */
+      int prev2_ea = 0;       /* EastAsian property of character before the previous character */
+      bool prev_initial_hyphen = false; /* the previous character was a
+                                           word-initial hyphen or U+2010 */
+      bool prev_nus = false; /* before the previous character, there was a character
                                 with line break property LBP_NU and since then
                                 only characters with line break property LBP_SY
                                 or LBP_IS */
-      int last_prop = LBP_BK; /* line break property of last non-space character */
+      int last_prop = LBP_BK; /* line break property of last non-space character
+                                 (= last character, ignoring intervening characters of class SP or CM or ZWJ) */
       char *seen_space = NULL; /* Was a space seen after the last non-space character? */
 
       /* Number of consecutive regional indicator (RI) characters seen
@@ -214,7 +217,7 @@ u32_possible_linebreaks_loop (const uint32_t *s, size_t n, const char *encoding,
                       /* (LB8) Break after zero-width space.  */
                       *p = UC_BREAK_POSSIBLE;
                     }
-                  else if (prev_prop == LBP_ZWJ)
+                  else if (preceding_prop == LBP_ZWJ)
                     {
                       /* (LB8a) Don't break right after a zero-width joiner.  */
                       *p = UC_BREAK_PROHIBITED;
@@ -361,21 +364,29 @@ u32_possible_linebreaks_loop (const uint32_t *s, size_t n, const char *encoding,
                   seen_space = NULL;
                 }
 
-              prev_initial_hyphen =
-                (prop == LBP_HY || uc == 0x2010)
-                && (prev_prop == LBP_BK || prev_prop == LBP_CR || prev_prop == LBP_LF
-                    || prev_prop == LBP_SP || prev_prop == LBP_ZW
-                    || prev_prop == LBP_CB || prev_prop == LBP_GL);
-              prev_prop = (prop == LBP_VI && (prev_prop == LBP_AK
-                                              || prev_prop == LBP_AL2
-                                              || prev_prop == LBP_AS)
-                           ? LBP_AKLS_VI :
-                           prev_prop == LBP_HL && (prop == LBP_HY
-                                                   || (prop == LBP_BA && !ea))
-                           ? LBP_HL_BA :
-                           prop);
-              prev2_ea = prev_ea;
-              prev_ea = ea;
+              /* (LB9) Treat X (CM | ZWJ)* as if it were X, where X is any line
+                 break class except BK, CR, LF, NL, SP, or ZW.  */
+              if (!((prop == LBP_CM || prop == LBP_ZWJ)
+                    && !(prev_prop == LBP_BK || prev_prop == LBP_LF || prev_prop == LBP_CR
+                         || prev_prop == LBP_SP || prev_prop == LBP_ZW)))
+                {
+                  prev_initial_hyphen =
+                    (prop == LBP_HY || uc == 0x2010)
+                    && (prev_prop == LBP_BK || prev_prop == LBP_CR || prev_prop == LBP_LF
+                        || prev_prop == LBP_SP || prev_prop == LBP_ZW
+                        || prev_prop == LBP_CB || prev_prop == LBP_GL);
+                  prev_prop = (prop == LBP_VI && (prev_prop == LBP_AK
+                                                  || prev_prop == LBP_AL2
+                                                  || prev_prop == LBP_AS)
+                               ? LBP_AKLS_VI :
+                               prev_prop == LBP_HL && (prop == LBP_HY
+                                                       || (prop == LBP_BA && !ea))
+                               ? LBP_HL_BA :
+                               prop);
+                  prev2_ea = prev_ea;
+                  prev_ea = ea;
+                }
+              preceding_prop = prop;
             }
 
           prev_nus = nus;
index 65248e5c403a51300600664f630124dd63826e62..6c26a8c26f2e6ed6890cf6386b30e7b6469062f7 100644 (file)
@@ -78,17 +78,20 @@ u8_possible_linebreaks_loop (const uint8_t *s, size_t n, const char *encoding,
           lookahead2_prop_ea = PROP_EA (LBP_BK, 0);
         }
 
-      int prev_prop = LBP_BK; /* line break property of last character */
-      int prev_ea = 0;        /* EastAsian property of last character */
-      int prev2_ea = 0;       /* EastAsian property of character before the
-                                 last character */
-      bool prev_initial_hyphen = false; /* the last character was a word-initial
-                                           hyphen or U+2010 */
-      bool prev_nus = false; /* before the last character, there was a character
+      int preceding_prop = LBP_BK; /* line break property of preceding character */
+      int prev_prop = LBP_BK; /* line break property of previous character
+                                 (= last character, ignoring intervening characters of class CM or ZWJ) */
+      int prev_ea = 0;        /* EastAsian property of previous character
+                                 (= last character, ignoring intervening characters of class CM or ZWJ) */
+      int prev2_ea = 0;       /* EastAsian property of character before the previous character */
+      bool prev_initial_hyphen = false; /* the previous character was a
+                                           word-initial hyphen or U+2010 */
+      bool prev_nus = false; /* before the previous character, there was a character
                                 with line break property LBP_NU and since then
                                 only characters with line break property LBP_SY
                                 or LBP_IS */
-      int last_prop = LBP_BK; /* line break property of last non-space character */
+      int last_prop = LBP_BK; /* line break property of last non-space character
+                                 (= last character, ignoring intervening characters of class SP or CM or ZWJ) */
       char *seen_space = NULL; /* Was a space seen after the last non-space character? */
 
       /* Number of consecutive regional indicator (RI) characters seen
@@ -221,7 +224,7 @@ u8_possible_linebreaks_loop (const uint8_t *s, size_t n, const char *encoding,
                       /* (LB8) Break after zero-width space.  */
                       *p = UC_BREAK_POSSIBLE;
                     }
-                  else if (prev_prop == LBP_ZWJ)
+                  else if (preceding_prop == LBP_ZWJ)
                     {
                       /* (LB8a) Don't break right after a zero-width joiner.  */
                       *p = UC_BREAK_PROHIBITED;
@@ -368,21 +371,29 @@ u8_possible_linebreaks_loop (const uint8_t *s, size_t n, const char *encoding,
                   seen_space = NULL;
                 }
 
-              prev_initial_hyphen =
-                (prop == LBP_HY || uc == 0x2010)
-                && (prev_prop == LBP_BK || prev_prop == LBP_CR || prev_prop == LBP_LF
-                    || prev_prop == LBP_SP || prev_prop == LBP_ZW
-                    || prev_prop == LBP_CB || prev_prop == LBP_GL);
-              prev_prop = (prop == LBP_VI && (prev_prop == LBP_AK
-                                              || prev_prop == LBP_AL2
-                                              || prev_prop == LBP_AS)
-                           ? LBP_AKLS_VI :
-                           prev_prop == LBP_HL && (prop == LBP_HY
-                                                   || (prop == LBP_BA && !ea))
-                           ? LBP_HL_BA :
-                           prop);
-              prev2_ea = prev_ea;
-              prev_ea = ea;
+              /* (LB9) Treat X (CM | ZWJ)* as if it were X, where X is any line
+                 break class except BK, CR, LF, NL, SP, or ZW.  */
+              if (!((prop == LBP_CM || prop == LBP_ZWJ)
+                    && !(prev_prop == LBP_BK || prev_prop == LBP_LF || prev_prop == LBP_CR
+                         || prev_prop == LBP_SP || prev_prop == LBP_ZW)))
+                {
+                  prev_initial_hyphen =
+                    (prop == LBP_HY || uc == 0x2010)
+                    && (prev_prop == LBP_BK || prev_prop == LBP_CR || prev_prop == LBP_LF
+                        || prev_prop == LBP_SP || prev_prop == LBP_ZW
+                        || prev_prop == LBP_CB || prev_prop == LBP_GL);
+                  prev_prop = (prop == LBP_VI && (prev_prop == LBP_AK
+                                                  || prev_prop == LBP_AL2
+                                                  || prev_prop == LBP_AS)
+                               ? LBP_AKLS_VI :
+                               prev_prop == LBP_HL && (prop == LBP_HY
+                                                       || (prop == LBP_BA && !ea))
+                               ? LBP_HL_BA :
+                               prop);
+                  prev2_ea = prev_ea;
+                  prev_ea = ea;
+                }
+              preceding_prop = prop;
             }
 
           prev_nus = nus;