+2021-12-28 Bruno Haible <bruno@clisp.org>
+
+ unilbrk: Update handling of zero-width joiner for Unicode 10.0.0.
+ * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Update
+ code for zero-width joiner handling to match UAX #14 for Unicode 10.0.0.
+ * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks):
+ Likewise.
+ * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks):
+ Likewise.
+ * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test
+ regarding zero-width joiner.
+ * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise.
+ * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise.
+
2021-12-28 Bruno Haible <bruno@clisp.org>
unilbrk: Update handling of combining marks for Unicode 10.0.0.
{
int LBP_AI_REPLACEMENT = (is_cjk_encoding (encoding) ? LBP_ID : LBP_AL);
const uint16_t *s_end = s + n;
+ int prev_prop = LBP_BK; /* line break property of last character */
int last_prop = LBP_BK; /* line break property of last non-space character */
char *seen_space = NULL; /* Was a space seen after the last non-space character? */
{
/* (LB4,LB5,LB6) Mandatory break. */
*p = UC_BREAK_MANDATORY;
+ prev_prop = LBP_BK;
last_prop = LBP_BK;
seen_space = NULL;
}
last_prop = LBP_ZW;
seen_space = NULL;
}
- else if (prop == LBP_CM)
+ else if (prop == LBP_CM || prop == LBP_ZWJ)
{
- /* (LB9) Don't break just before a combining character, except
- immediately after a mandatory break character, space, or
- zero-width space. */
+ /* (LB9) Don't break just before a combining character or
+ zero-width joiner, except immediately after a mandatory
+ break character, space, or zero-width space. */
if (last_prop == LBP_BK)
{
/* (LB4,LB5,LB6) Don't break at the beginning of a line. */
*p = UC_BREAK_PROHIBITED;
- /* Treat CM as AL. */
+ /* (LB10) Treat CM or ZWJ as AL. */
last_prop = LBP_AL;
seen_space = NULL;
}
character as base for combining marks" because now the
NBSP CM sequence is recommended instead of SP CM. */
*p = UC_BREAK_POSSIBLE;
- /* Treat CM as AL. */
+ /* (LB10) Treat CM or ZWJ as AL. */
last_prop = LBP_AL;
seen_space = NULL;
}
/* (LB8) Break after zero-width space. */
*p = UC_BREAK_POSSIBLE;
}
+ else if (prev_prop == LBP_ZWJ
+ && (prop == LBP_ID || prop == LBP_EB || prop == LBP_EM))
+ {
+ /* (LB8a) Don't break right after a zero-width joiner. */
+ *p = UC_BREAK_PROHIBITED;
+ }
else
{
switch (unilbrk_table [last_prop] [prop])
last_prop = prop;
seen_space = NULL;
}
+
+ prev_prop = prop;
}
s += count;
{
int LBP_AI_REPLACEMENT = (is_cjk_encoding (encoding) ? LBP_ID : LBP_AL);
const uint32_t *s_end = s + n;
+ int prev_prop = LBP_BK; /* line break property of last character */
int last_prop = LBP_BK; /* line break property of last non-space character */
char *seen_space = NULL; /* Was a space seen after the last non-space character? */
{
/* (LB4,LB5,LB6) Mandatory break. */
*p = UC_BREAK_MANDATORY;
+ prev_prop = LBP_BK;
last_prop = LBP_BK;
seen_space = NULL;
}
last_prop = LBP_ZW;
seen_space = NULL;
}
- else if (prop == LBP_CM)
+ else if (prop == LBP_CM || prop == LBP_ZWJ)
{
- /* (LB9) Don't break just before a combining character, except
- immediately after a mandatory break character, space, or
- zero-width space. */
+ /* (LB9) Don't break just before a combining character or
+ zero-width joiner, except immediately after a mandatory
+ break character, space, or zero-width space. */
if (last_prop == LBP_BK)
{
/* (LB4,LB5,LB6) Don't break at the beginning of a line. */
*p = UC_BREAK_PROHIBITED;
- /* Treat CM as AL. */
+ /* (LB10) Treat CM or ZWJ as AL. */
last_prop = LBP_AL;
seen_space = NULL;
}
character as base for combining marks" because now the
NBSP CM sequence is recommended instead of SP CM. */
*p = UC_BREAK_POSSIBLE;
- /* Treat CM as AL. */
+ /* (LB10) Treat CM or ZWJ as AL. */
last_prop = LBP_AL;
seen_space = NULL;
}
/* (LB8) Break after zero-width space. */
*p = UC_BREAK_POSSIBLE;
}
+ else if (prev_prop == LBP_ZWJ
+ && (prop == LBP_ID || prop == LBP_EB || prop == LBP_EM))
+ {
+ /* (LB8a) Don't break right after a zero-width joiner. */
+ *p = UC_BREAK_PROHIBITED;
+ }
else
{
switch (unilbrk_table [last_prop] [prop])
last_prop = prop;
seen_space = NULL;
}
+
+ prev_prop = prop;
}
s++;
{
int LBP_AI_REPLACEMENT = (is_cjk_encoding (encoding) ? LBP_ID : LBP_AL);
const uint8_t *s_end = s + n;
+ int prev_prop = LBP_BK; /* line break property of last character */
int last_prop = LBP_BK; /* line break property of last non-space character */
char *seen_space = NULL; /* Was a space seen after the last non-space character? */
{
/* (LB4,LB5,LB6) Mandatory break. */
*p = UC_BREAK_MANDATORY;
+ prev_prop = LBP_BK;
last_prop = LBP_BK;
seen_space = NULL;
}
last_prop = LBP_ZW;
seen_space = NULL;
}
- else if (prop == LBP_CM)
+ else if (prop == LBP_CM || prop == LBP_ZWJ)
{
- /* (LB9) Don't break just before a combining character, except
- immediately after a mandatory break character, space, or
- zero-width space. */
+ /* (LB9) Don't break just before a combining character or
+ zero-width joiner, except immediately after a mandatory
+ break character, space, or zero-width space. */
if (last_prop == LBP_BK)
{
/* (LB4,LB5,LB6) Don't break at the beginning of a line. */
*p = UC_BREAK_PROHIBITED;
- /* Treat CM as AL. */
+ /* (LB10) Treat CM or ZWJ as AL. */
last_prop = LBP_AL;
seen_space = NULL;
}
character as base for combining marks" because now the
NBSP CM sequence is recommended instead of SP CM. */
*p = UC_BREAK_POSSIBLE;
- /* Treat CM as AL. */
+ /* (LB10) Treat CM or ZWJ as AL. */
last_prop = LBP_AL;
seen_space = NULL;
}
/* (LB8) Break after zero-width space. */
*p = UC_BREAK_POSSIBLE;
}
+ else if (prev_prop == LBP_ZWJ
+ && (prop == LBP_ID || prop == LBP_EB || prop == LBP_EM))
+ {
+ /* (LB8a) Don't break right after a zero-width joiner. */
+ *p = UC_BREAK_PROHIBITED;
+ }
else
{
switch (unilbrk_table [last_prop] [prop])
last_prop = prop;
seen_space = NULL;
}
+
+ prev_prop = prop;
}
s += count;
free (p);
}
+ /* Test line breaking of zero-width joiners (U+200D). */
+ {
+ static const uint16_t input[39] =
+ {
+ 0x6709, 0x7121, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x6578, 0x3002, '\n',
+ 0x6709, 0x200D, 0x7121, 0x200D, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x200D, 0x6578, 0x3002, '\n',
+ 0x4F60, 0x2014, 0x4E0D, '\n',
+ 0x4F60, 0x2014, 0x200D, 0x4E0D, '\n',
+ 0x261D, 0xD83C, 0xDFFF, '\n',
+ 0x261D, 0x200D, 0xD83C, 0xDFFF, '\n',
+ };
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 39; i++)
+ {
+ ASSERT (p[i] == (i == 8 || i == 20
+ || i == 24 || i == 29
+ || i == 33 || i == 38 ? UC_BREAK_MANDATORY :
+ i == 1 || i == 2 || i == 3 || i == 4 || i == 5 || i == 6
+ || i == 14 || i == 15 || i == 16
+ || i == 22 || i == 23
+ || i == 26 ? UC_BREAK_POSSIBLE :
+ UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
return 0;
}
free (p);
}
+ /* Test line breaking of zero-width joiners (U+200D). */
+ {
+ static const uint32_t input[37] =
+ {
+ 0x6709, 0x7121, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x6578, 0x3002, '\n',
+ 0x6709, 0x200D, 0x7121, 0x200D, 0x7AAE, 0x591A, 0x500B, 0x7D20, 0x200D, 0x6578, 0x3002, '\n',
+ 0x4F60, 0x2014, 0x4E0D, '\n',
+ 0x4F60, 0x2014, 0x200D, 0x4E0D, '\n',
+ 0x261D, 0x1F3FF, '\n',
+ 0x261D, 0x200D, 0x1F3FF, '\n',
+ };
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 37; i++)
+ {
+ ASSERT (p[i] == (i == 8 || i == 20
+ || i == 24 || i == 29
+ || i == 32 || i == 36 ? UC_BREAK_MANDATORY :
+ i == 1 || i == 2 || i == 3 || i == 4 || i == 5 || i == 6
+ || i == 14 || i == 15 || i == 16
+ || i == 22 || i == 23
+ || i == 26 ? UC_BREAK_POSSIBLE :
+ UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
return 0;
}
free (p);
}
+ /* Test line breaking of zero-width joiners (U+200D). */
+ {
+ static const uint8_t input[101] =
+ "\346\234\211\347\204\241\347\252\256\345\244\232\345\200\213\347\264\240\346\225\270\343\200\202\n" /* "有無窮多個素數。" */
+ "\346\234\211\342\200\215\347\204\241\342\200\215\347\252\256\345\244\232\345\200\213\347\264\240\342\200\215\346\225\270\343\200\202\n"
+ "\344\275\240\342\200\224\344\270\215\n" /* "你—不" */
+ "\344\275\240\342\200\224\342\200\215\344\270\215\n"
+ "\342\230\235\360\237\217\277\n" /* "☝🏿" */
+ "\342\230\235\342\200\215\360\237\217\277\n";
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 101; i++)
+ {
+ ASSERT (p[i] == (i == 24 || i == 58
+ || i == 68 || i == 81
+ || i == 89 || i == 100 ? UC_BREAK_MANDATORY :
+ i == 3 || i == 6 || i == 9 || i == 12 || i == 15 || i == 18
+ || i == 40 || i == 43 || i == 46
+ || i == 62 || i == 65
+ || i == 72 ? UC_BREAK_POSSIBLE :
+ UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
return 0;
}