+2021-12-28 Bruno Haible <bruno@clisp.org>
+
+ unilbrk: Update handling of regional indicators for Unicode 10.0.0.
+ * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Add
+ code for handling regional indicators.
+ * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks):
+ Likewise.
+ * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks):
+ Likewise.
+ * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test
+ regarding regional indicators.
+ * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise.
+ * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise.
+
2021-12-28 Bruno Haible <bruno@clisp.org>
unilbrk: Update handling of zero-width joiner for Unicode 10.0.0.
/* Don't break inside multibyte characters. */
memset (p, UC_BREAK_PROHIBITED, n);
+ /* Number of consecutive regional indicator (RI) characters seen
+ immediately before the current point. */
+ size_t ri_count = 0;
+
do
{
ucs4_t uc;
/* (LB8a) Don't break right after a zero-width joiner. */
*p = UC_BREAK_PROHIBITED;
}
+ else if (last_prop == LBP_RI && prop == LBP_RI)
+ {
+ /* (LB30a) Break between two regional indicator symbols
+ if and only if there are an even number of regional
+ indicators preceding the position of the break. */
+ *p = (seen_space != NULL || (ri_count % 2) == 0
+ ? UC_BREAK_POSSIBLE
+ : UC_BREAK_PROHIBITED);
+ }
else
{
switch (unilbrk_table [last_prop] [prop])
prev_prop = prop;
}
+ if (prop == LBP_RI)
+ ri_count++;
+ else
+ ri_count = 0;
+
s += count;
p += count;
}
int last_prop = LBP_BK; /* line break property of last non-space character */
char *seen_space = NULL; /* Was a space seen after the last non-space character? */
+ /* Number of consecutive regional indicator (RI) characters seen
+ immediately before the current point. */
+ size_t ri_count = 0;
+
do
{
ucs4_t uc = *s;
/* (LB8a) Don't break right after a zero-width joiner. */
*p = UC_BREAK_PROHIBITED;
}
+ else if (last_prop == LBP_RI && prop == LBP_RI)
+ {
+ /* (LB30a) Break between two regional indicator symbols
+ if and only if there are an even number of regional
+ indicators preceding the position of the break. */
+ *p = (seen_space != NULL || (ri_count % 2) == 0
+ ? UC_BREAK_POSSIBLE
+ : UC_BREAK_PROHIBITED);
+ }
else
{
switch (unilbrk_table [last_prop] [prop])
prev_prop = prop;
}
+ if (prop == LBP_RI)
+ ri_count++;
+ else
+ ri_count = 0;
+
s++;
p++;
}
/* Don't break inside multibyte characters. */
memset (p, UC_BREAK_PROHIBITED, n);
+ /* Number of consecutive regional indicator (RI) characters seen
+ immediately before the current point. */
+ size_t ri_count = 0;
+
do
{
ucs4_t uc;
/* (LB8a) Don't break right after a zero-width joiner. */
*p = UC_BREAK_PROHIBITED;
}
+ else if (last_prop == LBP_RI && prop == LBP_RI)
+ {
+ /* (LB30a) Break between two regional indicator symbols
+ if and only if there are an even number of regional
+ indicators preceding the position of the break. */
+ *p = (seen_space != NULL || (ri_count % 2) == 0
+ ? UC_BREAK_POSSIBLE
+ : UC_BREAK_PROHIBITED);
+ }
else
{
switch (unilbrk_table [last_prop] [prop])
prev_prop = prop;
}
+ if (prop == LBP_RI)
+ ri_count++;
+ else
+ ri_count = 0;
+
s += count;
p += count;
}
free (p);
}
+ /* Test line breaking of regional indicators. */
+ {
+ static const uint16_t input[8] =
+ { 0xD83C, 0xDDE9, 0xD83C, 0xDDEA, 0xD83C, 0xDDEB, 0xD83C, 0xDDF7 };
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 8; i++)
+ {
+ ASSERT (p[i] == (i == 4 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
return 0;
}
free (p);
}
+ /* Test line breaking of regional indicators. */
+ {
+ static const uint32_t input[4] =
+ { 0x1F1E9, 0x1F1EA, 0x1F1EB, 0x1F1F7 };
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 4; i++)
+ {
+ ASSERT (p[i] == (i == 2 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
return 0;
}
free (p);
}
+ /* Test line breaking of regional indicators. */
+ {
+ static const uint8_t input[16] =
+ "\360\237\207\251\360\237\207\252\360\237\207\253\360\237\207\267";
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 16; i++)
+ {
+ ASSERT (p[i] == (i == 8 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
return 0;
}