]> Savannah Git Hosting - gnulib.git/commitdiff
unilbrk: Update handling of regional indicators for Unicode 10.0.0.
authorBruno Haible <bruno@clisp.org>
Tue, 28 Dec 2021 23:06:17 +0000 (00:06 +0100)
committerBruno Haible <bruno@clisp.org>
Tue, 28 Dec 2021 23:06:17 +0000 (00:06 +0100)
* lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Add
code for handling regional indicators.
* lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks):
Likewise.
* lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks):
Likewise.
* tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test
regarding regional indicators.
* tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise.
* tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise.

ChangeLog
lib/unilbrk/u16-possible-linebreaks.c
lib/unilbrk/u32-possible-linebreaks.c
lib/unilbrk/u8-possible-linebreaks.c
tests/unilbrk/test-u16-possible-linebreaks.c
tests/unilbrk/test-u32-possible-linebreaks.c
tests/unilbrk/test-u8-possible-linebreaks.c

index 1ae731cf10fa2bf282eebda803af9bae9d74e612..74a988abd233e29b678b582a32bac22b26d979f3 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2021-12-28  Bruno Haible  <bruno@clisp.org>
+
+       unilbrk: Update handling of regional indicators for Unicode 10.0.0.
+       * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Add
+       code for handling regional indicators.
+       * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks):
+       Likewise.
+       * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks):
+       Likewise.
+       * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test
+       regarding regional indicators.
+       * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise.
+       * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise.
+
 2021-12-28  Bruno Haible  <bruno@clisp.org>
 
        unilbrk: Update handling of zero-width joiner for Unicode 10.0.0.
index e0f0ff06b74b6157acb0c6bf0753c46c45d62ff4..7ad88367c7cf60f18bc82ee1fc49f2393edb3d41 100644 (file)
@@ -49,6 +49,10 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char
       /* Don't break inside multibyte characters.  */
       memset (p, UC_BREAK_PROHIBITED, n);
 
+      /* Number of consecutive regional indicator (RI) characters seen
+         immediately before the current point.  */
+      size_t ri_count = 0;
+
       do
         {
           ucs4_t uc;
@@ -152,6 +156,15 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char
                       /* (LB8a) Don't break right after a zero-width joiner.  */
                       *p = UC_BREAK_PROHIBITED;
                     }
+                  else if (last_prop == LBP_RI && prop == LBP_RI)
+                    {
+                      /* (LB30a) Break between two regional indicator symbols
+                         if and only if there are an even number of regional
+                         indicators preceding the position of the break.  */
+                      *p = (seen_space != NULL || (ri_count % 2) == 0
+                            ? UC_BREAK_POSSIBLE
+                            : UC_BREAK_PROHIBITED);
+                    }
                   else
                     {
                       switch (unilbrk_table [last_prop] [prop])
@@ -176,6 +189,11 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char
               prev_prop = prop;
             }
 
+          if (prop == LBP_RI)
+            ri_count++;
+          else
+            ri_count = 0;
+
           s += count;
           p += count;
         }
index ab973e911c80e79192e826b17ace7aaaabba28e5..0c84ff4f1a797f186a7ee4a05a7eecc1ae3f142a 100644 (file)
@@ -44,6 +44,10 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char
       int last_prop = LBP_BK; /* line break property of last non-space character */
       char *seen_space = NULL; /* Was a space seen after the last non-space character? */
 
+      /* Number of consecutive regional indicator (RI) characters seen
+         immediately before the current point.  */
+      size_t ri_count = 0;
+
       do
         {
           ucs4_t uc = *s;
@@ -146,6 +150,15 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char
                       /* (LB8a) Don't break right after a zero-width joiner.  */
                       *p = UC_BREAK_PROHIBITED;
                     }
+                  else if (last_prop == LBP_RI && prop == LBP_RI)
+                    {
+                      /* (LB30a) Break between two regional indicator symbols
+                         if and only if there are an even number of regional
+                         indicators preceding the position of the break.  */
+                      *p = (seen_space != NULL || (ri_count % 2) == 0
+                            ? UC_BREAK_POSSIBLE
+                            : UC_BREAK_PROHIBITED);
+                    }
                   else
                     {
                       switch (unilbrk_table [last_prop] [prop])
@@ -170,6 +183,11 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char
               prev_prop = prop;
             }
 
+          if (prop == LBP_RI)
+            ri_count++;
+          else
+            ri_count = 0;
+
           s++;
           p++;
         }
index 86f1ce8f1388a6c65fa6e3257dd159c7410bbeaf..b631ddeec62d90cbeb5d48243ab3d085cf4aef82 100644 (file)
@@ -49,6 +49,10 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char *
       /* Don't break inside multibyte characters.  */
       memset (p, UC_BREAK_PROHIBITED, n);
 
+      /* Number of consecutive regional indicator (RI) characters seen
+         immediately before the current point.  */
+      size_t ri_count = 0;
+
       do
         {
           ucs4_t uc;
@@ -152,6 +156,15 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char *
                       /* (LB8a) Don't break right after a zero-width joiner.  */
                       *p = UC_BREAK_PROHIBITED;
                     }
+                  else if (last_prop == LBP_RI && prop == LBP_RI)
+                    {
+                      /* (LB30a) Break between two regional indicator symbols
+                         if and only if there are an even number of regional
+                         indicators preceding the position of the break.  */
+                      *p = (seen_space != NULL || (ri_count % 2) == 0
+                            ? UC_BREAK_POSSIBLE
+                            : UC_BREAK_PROHIBITED);
+                    }
                   else
                     {
                       switch (unilbrk_table [last_prop] [prop])
@@ -176,6 +189,11 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char *
               prev_prop = prop;
             }
 
+          if (prop == LBP_RI)
+            ri_count++;
+          else
+            ri_count = 0;
+
           s += count;
           p += count;
         }
index d8dd863472aecdaada8ec0eb89f8544e2330bed3..87d712fcc6422e5429d85720316cb15e89fee27e 100644 (file)
@@ -164,5 +164,20 @@ main ()
     free (p);
   }
 
+  /* Test line breaking of regional indicators.  */
+  {
+    static const uint16_t input[8] =
+      { 0xD83C, 0xDDE9, 0xD83C, 0xDDEA, 0xD83C, 0xDDEB, 0xD83C, 0xDDF7 };
+    char *p = (char *) malloc (SIZEOF (input));
+    size_t i;
+
+    u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+    for (i = 0; i < 8; i++)
+      {
+        ASSERT (p[i] == (i == 4 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
+      }
+    free (p);
+  }
+
   return 0;
 }
index 97c08ad5a99d91918f721d09b062ef0186ed1095..c140ca3b55463a9268a2d6d96aef80470c9aaca6 100644 (file)
@@ -164,5 +164,20 @@ main ()
     free (p);
   }
 
+  /* Test line breaking of regional indicators.  */
+  {
+    static const uint32_t input[4] =
+      { 0x1F1E9, 0x1F1EA, 0x1F1EB, 0x1F1F7 };
+    char *p = (char *) malloc (SIZEOF (input));
+    size_t i;
+
+    u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+    for (i = 0; i < 4; i++)
+      {
+        ASSERT (p[i] == (i == 2 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
+      }
+    free (p);
+  }
+
   return 0;
 }
index 9dd566695402692753817603d7c58868e48aaeb9..0fdb66d3678407b9163a8fdb3b9db0ad583dfb83 100644 (file)
@@ -149,5 +149,20 @@ main ()
     free (p);
   }
 
+  /* Test line breaking of regional indicators.  */
+  {
+    static const uint8_t input[16] =
+      "\360\237\207\251\360\237\207\252\360\237\207\253\360\237\207\267";
+    char *p = (char *) malloc (SIZEOF (input));
+    size_t i;
+
+    u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+    for (i = 0; i < 16; i++)
+      {
+        ASSERT (p[i] == (i == 8 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
+      }
+    free (p);
+  }
+
   return 0;
 }