From: Bruno Haible Date: Tue, 28 Dec 2021 23:06:00 +0000 (+0100) Subject: unilbrk: Restore deviation for (IS,AL) pair. X-Git-Tag: v1.0~2462 X-Git-Url: https://gitweb.git.savannah.gnu.org/gitweb/?a=commitdiff_plain;h=c8d8a8ed6f782efbcbdbe90467050c0eb9fb76b3;p=gnulib.git unilbrk: Restore deviation for (IS,AL) pair. * lib/gen-uni-tables.c (output_lbrk_rules_as_tables): Disable rule LB29. * lib/unilbrk/lbrktables.c: Regenerated. * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add test case with HTML markup. * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-ulc-possible-linebreaks.c (main): Likewise. --- diff --git a/ChangeLog b/ChangeLog index 9f5cd5b6e6..193dd39210 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2021-12-28 Bruno Haible + + unilbrk: Restore deviation for (IS,AL) pair. + * lib/gen-uni-tables.c (output_lbrk_rules_as_tables): Disable rule LB29. + * lib/unilbrk/lbrktables.c: Regenerated. + * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add test case with + HTML markup. + * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-ulc-possible-linebreaks.c (main): Likewise. + 2021-12-28 Bruno Haible unilbrk: Change algorithm to be closer to Unicode Standard Annex #14. diff --git a/lib/gen-uni-tables.c b/lib/gen-uni-tables.c index d9f81943b5..3d1b718067 100644 --- a/lib/gen-uni-tables.c +++ b/lib/gen-uni-tables.c @@ -8149,8 +8149,12 @@ output_lbrk_rules_as_tables (const char *filename, const char *version) /* (LB29) Do not break between numeric punctuation and alphabetics ("e.g."). */ + /* We don't implement this rule, because we find it desirable to break before + the HTML tag "

" in strings like "

Some sentence.

". */ +#if 0 before = LBP_IS; after = LBP_AL; table[before][after].prohibited_no_sp = true; before = LBP_IS; after = LBP_HL; table[before][after].prohibited_no_sp = true; +#endif /* (LB28) Do not break between alphabetics ("at"). */ before = LBP_AL; after = LBP_AL; table[before][after].prohibited_no_sp = true; diff --git a/lib/unilbrk/lbrktables.c b/lib/unilbrk/lbrktables.c index dc6b3931cf..fd72a5cd84 100644 --- a/lib/unilbrk/lbrktables.c +++ b/lib/unilbrk/lbrktables.c @@ -50,7 +50,7 @@ const unsigned char unilbrk_table[30][30] = /* NS */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, /* OP */ { P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, }, /* QU */ { P, I, I, I, I, I, P, P, P, I, I, P, I, P, I, I, I, P, I, I, I, I, I, I, I, I, I, I, I, I, }, -/* IS */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, I, D, D, D, D, D, D, I, D, I, D, D, }, +/* IS */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, /* NU */ { P, I, D, I, D, I, P, P, P, I, I, I, I, P, I, I, I, P, I, D, D, D, D, D, D, I, D, I, D, D, }, /* PO */ { P, I, D, I, D, I, P, P, P, D, I, I, I, P, I, D, D, P, I, D, D, D, D, D, D, I, D, I, D, D, }, /* PR */ { P, I, D, I, D, I, P, P, P, D, I, I, I, P, I, D, D, P, I, I, I, I, I, I, I, I, D, I, I, I, }, diff --git a/tests/unilbrk/test-u16-possible-linebreaks.c b/tests/unilbrk/test-u16-possible-linebreaks.c index b27d75b2bc..c8c5c50a7a 100644 --- a/tests/unilbrk/test-u16-possible-linebreaks.c +++ b/tests/unilbrk/test-u16-possible-linebreaks.c @@ -95,5 +95,24 @@ main () free (p); } + /* Test line breaking in a string with HTML markup. */ + { + static const uint16_t input[21] = + { + '<', 'P', '>', 'S', 'o', 'm', 'e', ' ', 's', 'e', 'n', 't', + 'e', 'n', 'c', 'e', '.', '<', '/', 'P', '>' + }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 21; i++) + { + ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u32-possible-linebreaks.c b/tests/unilbrk/test-u32-possible-linebreaks.c index 1474ae1c27..b41f49cf2d 100644 --- a/tests/unilbrk/test-u32-possible-linebreaks.c +++ b/tests/unilbrk/test-u32-possible-linebreaks.c @@ -95,5 +95,24 @@ main () free (p); } + /* Test line breaking in a string with HTML markup. */ + { + static const uint32_t input[21] = + { + '<', 'P', '>', 'S', 'o', 'm', 'e', ' ', 's', 'e', 'n', 't', + 'e', 'n', 'c', 'e', '.', '<', '/', 'P', '>' + }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 21; i++) + { + ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u8-possible-linebreaks.c b/tests/unilbrk/test-u8-possible-linebreaks.c index c55e900696..b07464c289 100644 --- a/tests/unilbrk/test-u8-possible-linebreaks.c +++ b/tests/unilbrk/test-u8-possible-linebreaks.c @@ -82,12 +82,27 @@ main () size_t i; u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); - for (i = 0; i < 4; i++) + for (i = 0; i < 6; i++) { ASSERT (p[i] == (i == 5 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); } free (p); } + /* Test line breaking in a string with HTML markup. */ + { + static const uint8_t input[21] = "

Some sentence.

"; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 21; i++) + { + ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-ulc-possible-linebreaks.c b/tests/unilbrk/test-ulc-possible-linebreaks.c index 24bae4b717..f3ad7f9222 100644 --- a/tests/unilbrk/test-ulc-possible-linebreaks.c +++ b/tests/unilbrk/test-ulc-possible-linebreaks.c @@ -48,6 +48,21 @@ main () } free (p); } + + /* Test line breaking in a string with HTML markup. */ + { + static const char input[21] = "

Some sentence.

"; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + ulc_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 21; i++) + { + ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } #endif return 0;