From ad814f4ddc420d5a05dc489c7c8086fe29d76ecb Mon Sep 17 00:00:00 2001 From: Bruno Haible Date: Wed, 29 Dec 2021 09:12:34 +0100 Subject: [PATCH] unilbrk: Update handling of Hebrew letter + hyphen for Unicode 10.0.0. * lib/unilbrk/lbrktables.h (LBP_HL_BA): New enum value. * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Add code for handling break-after character after Hebrew letter. * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks): Likewise. * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks): Likewise. * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test regarding break-after character after Hebrew letter. * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. --- ChangeLog | 15 +++++++++++++++ lib/unilbrk/lbrktables.h | 4 +++- lib/unilbrk/u16-possible-linebreaks.c | 9 ++++++++- lib/unilbrk/u32-possible-linebreaks.c | 9 ++++++++- lib/unilbrk/u8-possible-linebreaks.c | 9 ++++++++- tests/unilbrk/test-u16-possible-linebreaks.c | 17 +++++++++++++++++ tests/unilbrk/test-u32-possible-linebreaks.c | 17 +++++++++++++++++ tests/unilbrk/test-u8-possible-linebreaks.c | 17 +++++++++++++++++ 8 files changed, 93 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index ed9e63b851..1ef0bc80ed 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,18 @@ +2021-12-29 Bruno Haible + + unilbrk: Update handling of Hebrew letter + hyphen for Unicode 10.0.0. + * lib/unilbrk/lbrktables.h (LBP_HL_BA): New enum value. + * lib/unilbrk/u8-possible-linebreaks.c (u8_possible_linebreaks): Add + code for handling break-after character after Hebrew letter. + * lib/unilbrk/u16-possible-linebreaks.c (u16_possible_linebreaks): + Likewise. + * lib/unilbrk/u32-possible-linebreaks.c (u32_possible_linebreaks): + Likewise. + * tests/unilbrk/test-u8-possible-linebreaks.c (main): Add a test + regarding break-after character after Hebrew letter. + * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. + 2021-12-28 Paul Eggert nstrftime: improve handling of invalid formats diff --git a/lib/unilbrk/lbrktables.h b/lib/unilbrk/lbrktables.h index 9c8f0bb745..31ce91ac17 100644 --- a/lib/unilbrk/lbrktables.h +++ b/lib/unilbrk/lbrktables.h @@ -72,7 +72,9 @@ enum LBP_ZWJ = 27, /* zero width joiner */ LBP_EB = 28, /* emoji base */ LBP_EM = 29, /* emoji modifier */ - LBP_XX = 37 /* unknown */ + LBP_XX = 37, /* unknown */ + /* Artificial values that exist only at runtime, not in the tables. */ + LBP_HL_BA = 100 }; #include "lbrkprop1.h" diff --git a/lib/unilbrk/u16-possible-linebreaks.c b/lib/unilbrk/u16-possible-linebreaks.c index 7ad88367c7..ee11730450 100644 --- a/lib/unilbrk/u16-possible-linebreaks.c +++ b/lib/unilbrk/u16-possible-linebreaks.c @@ -165,6 +165,11 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED); } + else if (prev_prop == LBP_HL_BA) + { + /* (LB21a) Don't break after Hebrew + Hyphen/Break-After. */ + *p = UC_BREAK_PROHIBITED; + } else { switch (unilbrk_table [last_prop] [prop]) @@ -186,7 +191,9 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char seen_space = NULL; } - prev_prop = prop; + prev_prop = (prev_prop == LBP_HL && (prop == LBP_HY || prop == LBP_BA) + ? LBP_HL_BA + : prop); } if (prop == LBP_RI) diff --git a/lib/unilbrk/u32-possible-linebreaks.c b/lib/unilbrk/u32-possible-linebreaks.c index 0c84ff4f1a..d36e2db882 100644 --- a/lib/unilbrk/u32-possible-linebreaks.c +++ b/lib/unilbrk/u32-possible-linebreaks.c @@ -159,6 +159,11 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED); } + else if (prev_prop == LBP_HL_BA) + { + /* (LB21a) Don't break after Hebrew + Hyphen/Break-After. */ + *p = UC_BREAK_PROHIBITED; + } else { switch (unilbrk_table [last_prop] [prop]) @@ -180,7 +185,9 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char seen_space = NULL; } - prev_prop = prop; + prev_prop = (prev_prop == LBP_HL && (prop == LBP_HY || prop == LBP_BA) + ? LBP_HL_BA + : prop); } if (prop == LBP_RI) diff --git a/lib/unilbrk/u8-possible-linebreaks.c b/lib/unilbrk/u8-possible-linebreaks.c index b631ddeec6..06b914af91 100644 --- a/lib/unilbrk/u8-possible-linebreaks.c +++ b/lib/unilbrk/u8-possible-linebreaks.c @@ -165,6 +165,11 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED); } + else if (prev_prop == LBP_HL_BA) + { + /* (LB21a) Don't break after Hebrew + Hyphen/Break-After. */ + *p = UC_BREAK_PROHIBITED; + } else { switch (unilbrk_table [last_prop] [prop]) @@ -186,7 +191,9 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * seen_space = NULL; } - prev_prop = prop; + prev_prop = (prev_prop == LBP_HL && (prop == LBP_HY || prop == LBP_BA) + ? LBP_HL_BA + : prop); } if (prop == LBP_RI) diff --git a/tests/unilbrk/test-u16-possible-linebreaks.c b/tests/unilbrk/test-u16-possible-linebreaks.c index 87d712fcc6..fedaa9d842 100644 --- a/tests/unilbrk/test-u16-possible-linebreaks.c +++ b/tests/unilbrk/test-u16-possible-linebreaks.c @@ -179,5 +179,22 @@ main () free (p); } + /* Test special behaviour of hyphen/break-after character after + Hebrew letter. */ + { + static const uint16_t input[10] = /* "ab-אב-αβ-ω" */ + { 'a', 'b', '-', 0x05D0, 0x05D1, '-', 0x03B1, 0x03B2, '-', 0x03C9 }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 10; i++) + { + ASSERT (p[i] == (i == 3 || i == 9 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u32-possible-linebreaks.c b/tests/unilbrk/test-u32-possible-linebreaks.c index c140ca3b55..1d8dfb122b 100644 --- a/tests/unilbrk/test-u32-possible-linebreaks.c +++ b/tests/unilbrk/test-u32-possible-linebreaks.c @@ -179,5 +179,22 @@ main () free (p); } + /* Test special behaviour of hyphen/break-after character after + Hebrew letter. */ + { + static const uint32_t input[10] = /* "ab-אב-αβ-ω" */ + { 'a', 'b', '-', 0x05D0, 0x05D1, '-', 0x03B1, 0x03B2, '-', 0x03C9 }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 10; i++) + { + ASSERT (p[i] == (i == 3 || i == 9 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } diff --git a/tests/unilbrk/test-u8-possible-linebreaks.c b/tests/unilbrk/test-u8-possible-linebreaks.c index 0fdb66d367..cad7b7afa3 100644 --- a/tests/unilbrk/test-u8-possible-linebreaks.c +++ b/tests/unilbrk/test-u8-possible-linebreaks.c @@ -164,5 +164,22 @@ main () free (p); } + /* Test special behaviour of hyphen/break-after character after + Hebrew letter. */ + { + static const uint8_t input[15] = /* "ab-אב-αβ-ω" */ + "ab-\327\220\327\221-\316\261\316\262-\317\211"; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 15; i++) + { + ASSERT (p[i] == (i == 3 || i == 13 ? UC_BREAK_POSSIBLE : + UC_BREAK_PROHIBITED)); + } + free (p); + } + return 0; } -- 2.39.5