From d798516dca5a6a6bccf79806c267f8406b0f0757 Mon Sep 17 00:00:00 2001 From: Bruno Haible Date: Wed, 29 Dec 2021 00:05:52 +0100 Subject: [PATCH] unilbrk: Change algorithm to be closer to Unicode Standard Annex #14. * lib/gen-uni-tables.c (lbp_value_to_string): New function, extracted from output_lbp. (output_lbp): Invoke it. (output_lbrk_rules_as_tables): New function. (main): Invoke it. * lib/unilbrk/lbrktables.c: Now automatically generated. * tests/unilbrk/test-u8-possible-linebreaks.c (main): Update expected result. * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-ulc-possible-linebreaks.c (main): Likewise. * tests/unilbrk/test-ulc-width-linebreaks.c (main): Likewise. --- ChangeLog | 16 + lib/gen-uni-tables.c | 442 +++++++++++++++++-- lib/unilbrk/lbrktables.c | 59 ++- tests/unilbrk/test-u16-possible-linebreaks.c | 4 +- tests/unilbrk/test-u32-possible-linebreaks.c | 4 +- tests/unilbrk/test-u8-possible-linebreaks.c | 4 +- tests/unilbrk/test-ulc-possible-linebreaks.c | 2 +- tests/unilbrk/test-ulc-width-linebreaks.c | 2 +- 8 files changed, 446 insertions(+), 87 deletions(-) diff --git a/ChangeLog b/ChangeLog index 75efed1960..9f5cd5b6e6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,19 @@ +2021-12-28 Bruno Haible + + unilbrk: Change algorithm to be closer to Unicode Standard Annex #14. + * lib/gen-uni-tables.c (lbp_value_to_string): New function, extracted + from output_lbp. + (output_lbp): Invoke it. + (output_lbrk_rules_as_tables): New function. + (main): Invoke it. + * lib/unilbrk/lbrktables.c: Now automatically generated. + * tests/unilbrk/test-u8-possible-linebreaks.c (main): Update expected + result. + * tests/unilbrk/test-u16-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-u32-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-ulc-possible-linebreaks.c (main): Likewise. + * tests/unilbrk/test-ulc-width-linebreaks.c (main): Likewise. + 2021-12-28 Bruno Haible uniwbrk: Add unit tests of regional indicators. diff --git a/lib/gen-uni-tables.c b/lib/gen-uni-tables.c index 854267320a..d9f81943b5 100644 --- a/lib/gen-uni-tables.c +++ b/lib/gen-uni-tables.c @@ -7856,6 +7856,59 @@ debug_output_org_lbrk_tables (const char *filename) } } +/* Given an enum value LBP_..., returns its name "LBP_..." as a string. */ +static const char * +lbp_value_to_string (unsigned int value) +{ + const char *value_string; + switch (value) + { +#define CASE(x) case x: value_string = #x; break; + CASE(LBP_BK); + CASE(LBP_CM); + CASE(LBP_WJ); + CASE(LBP_ZW); + CASE(LBP_GL); + CASE(LBP_SP); + CASE(LBP_B2); + CASE(LBP_BA); + CASE(LBP_BB); + CASE(LBP_HY); + CASE(LBP_CB); + CASE(LBP_CL); + CASE(LBP_CP); + CASE(LBP_EX); + CASE(LBP_IN); + CASE(LBP_NS); + CASE(LBP_OP); + CASE(LBP_QU); + CASE(LBP_IS); + CASE(LBP_NU); + CASE(LBP_PO); + CASE(LBP_PR); + CASE(LBP_SY); + CASE(LBP_AI); + CASE(LBP_AL); + CASE(LBP_H2); + CASE(LBP_H3); + CASE(LBP_HL); + CASE(LBP_ID); + CASE(LBP_JL); + CASE(LBP_JV); + CASE(LBP_JT); + CASE(LBP_RI); + CASE(LBP_SA); + CASE(LBP_ZWJ); + CASE(LBP_EB); + CASE(LBP_EM); + CASE(LBP_XX); +#undef CASE + default: + abort (); + } + return value_string; +} + /* Construction of sparse 3-level tables. */ #define TABLE lbp_table #define ELEMENT unsigned char @@ -7964,55 +8017,9 @@ output_lbp (FILE *stream1, FILE *stream2) for (i = 0; i < t.level3_size << t.p; i++) { unsigned char value = ((unsigned char *) (t.result + level3_offset))[i]; - const char *value_string; - switch (value) - { -#define CASE(x) case x: value_string = #x; break; - CASE(LBP_BK); - CASE(LBP_CM); - CASE(LBP_WJ); - CASE(LBP_ZW); - CASE(LBP_GL); - CASE(LBP_SP); - CASE(LBP_B2); - CASE(LBP_BA); - CASE(LBP_BB); - CASE(LBP_HY); - CASE(LBP_CB); - CASE(LBP_CL); - CASE(LBP_CP); - CASE(LBP_EX); - CASE(LBP_IN); - CASE(LBP_NS); - CASE(LBP_OP); - CASE(LBP_QU); - CASE(LBP_IS); - CASE(LBP_NU); - CASE(LBP_PO); - CASE(LBP_PR); - CASE(LBP_SY); - CASE(LBP_AI); - CASE(LBP_AL); - CASE(LBP_H2); - CASE(LBP_H3); - CASE(LBP_HL); - CASE(LBP_ID); - CASE(LBP_JL); - CASE(LBP_JV); - CASE(LBP_JT); - CASE(LBP_RI); - CASE(LBP_SA); - CASE(LBP_ZWJ); - CASE(LBP_EB); - CASE(LBP_EM); - CASE(LBP_XX); -#undef CASE - default: - abort (); - } if (i > 0 && (i % 8) == 0) fprintf (stream2, "\n "); - fprintf (stream2, " %s%s", value_string, + fprintf (stream2, " %s%s", lbp_value_to_string (value), (i+1 < t.level3_size << t.p ? "," : "")); } if (t.level3_size << t.p > 8) @@ -8069,6 +8076,346 @@ output_lbrk_tables (const char *filename1, const char *filename2, const char *ve } } +static void +output_lbrk_rules_as_tables (const char *filename, const char *version) +{ + FILE *stream; + + stream = fopen (filename, "w"); + if (stream == NULL) + { + fprintf (stderr, "cannot open '%s' for writing\n", filename); + exit (1); + } + + fprintf (stream, "/* DO NOT EDIT! GENERATED AUTOMATICALLY! */\n"); + fprintf (stream, "/* Table that encodes several line breaking rules. */\n"); + fprintf (stream, "/* Generated automatically by gen-uni-tables.c for Unicode %s. */\n", + version); + fprintf (stream, "\n"); + + fprintf (stream, "/* Copyright (C) 2001-2021 Free Software Foundation, Inc.\n"); + fprintf (stream, "\n"); + output_library_license (stream, false); + fprintf (stream, "\n"); + + fprintf (stream, "#include \n"); + fprintf (stream, "\n"); + fprintf (stream, "/* Specification. */\n"); + fprintf (stream, "#include \"unilbrk/lbrktables.h\"\n"); + fprintf (stream, "\n"); + fprintf (stream, "/* Define unilbrkprop, table of line breaking properties. */\n"); + fprintf (stream, "#include \"unilbrk/lbrkprop2.h\"\n"); + fprintf (stream, "\n"); + + /* LBP_* table indices are in the range 0 .. NLBP-1. */ + const unsigned int NLBP = 30; + + unsigned int before; + unsigned int after; + /* Describe the table cell (before, after). */ + struct table_cell + { + /* Break prohibited when no spaces, i.e. in before ÷ after */ + bool prohibited_no_sp; + /* Break prohibited with spaces, i.e. in before SP+ ÷ after */ + bool prohibited_with_sp; + }; + struct table_cell table[NLBP][NLBP]; + + /* Fill the table. + If we were to apply the rules in top-down order (high precedence rules + first), the table_cell fields have to support values false/true/unknown. + If we apply the rules in the opposite order (high precedence order last), + the table_cell fields need to support only the values false/true. + So, that's what we do here. */ + + /* (LB31) Break everywhere. */ + for (before = 0; before < NLBP; before++) + for (after = 0; after < NLBP; after++) + table[before][after].prohibited_no_sp = false; + + /* (LB30b) Do not break between an emoji base and an emoji modifier. */ + before = LBP_EB; after = LBP_EM; table[before][after].prohibited_no_sp = true; + + /* (LB30) Do not break between letters, numbers, or ordinary symbols and + opening or closing parentheses. */ + before = LBP_AL; after = LBP_OP; table[before][after].prohibited_no_sp = true; + before = LBP_HL; after = LBP_OP; table[before][after].prohibited_no_sp = true; + before = LBP_NU; after = LBP_OP; table[before][after].prohibited_no_sp = true; + before = LBP_CP; after = LBP_AL; table[before][after].prohibited_no_sp = true; + before = LBP_CP; after = LBP_HL; table[before][after].prohibited_no_sp = true; + before = LBP_CP; after = LBP_NU; table[before][after].prohibited_no_sp = true; + + /* (LB29) Do not break between numeric punctuation and alphabetics + ("e.g."). */ + before = LBP_IS; after = LBP_AL; table[before][after].prohibited_no_sp = true; + before = LBP_IS; after = LBP_HL; table[before][after].prohibited_no_sp = true; + + /* (LB28) Do not break between alphabetics ("at"). */ + before = LBP_AL; after = LBP_AL; table[before][after].prohibited_no_sp = true; + before = LBP_AL; after = LBP_HL; table[before][after].prohibited_no_sp = true; + before = LBP_HL; after = LBP_AL; table[before][after].prohibited_no_sp = true; + before = LBP_HL; after = LBP_HL; table[before][after].prohibited_no_sp = true; + + /* (LB27) Korean Syllable Block. */ + before = LBP_JL; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_JV; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_JT; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_H2; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_H3; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_JL; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_JV; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_JT; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_H2; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_H3; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_JL; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_JV; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_JT; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_H2; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_H3; table[before][after].prohibited_no_sp = true; + + /* (LB26) Do not break a Korean syllable. */ + before = LBP_JL; after = LBP_JL; table[before][after].prohibited_no_sp = true; + before = LBP_JL; after = LBP_JV; table[before][after].prohibited_no_sp = true; + before = LBP_JL; after = LBP_H2; table[before][after].prohibited_no_sp = true; + before = LBP_JL; after = LBP_H3; table[before][after].prohibited_no_sp = true; + before = LBP_JV; after = LBP_JV; table[before][after].prohibited_no_sp = true; + before = LBP_JV; after = LBP_JT; table[before][after].prohibited_no_sp = true; + before = LBP_H2; after = LBP_JV; table[before][after].prohibited_no_sp = true; + before = LBP_H2; after = LBP_JT; table[before][after].prohibited_no_sp = true; + before = LBP_JT; after = LBP_JT; table[before][after].prohibited_no_sp = true; + before = LBP_H3; after = LBP_JT; table[before][after].prohibited_no_sp = true; + + /* (LB25) Do not break between the following pairs of classes relevant to + numbers. */ + before = LBP_CL; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_CP; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_CL; after = LBP_PR; table[before][after].prohibited_no_sp = true; + before = LBP_CP; after = LBP_PR; table[before][after].prohibited_no_sp = true; + before = LBP_NU; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_NU; after = LBP_PR; table[before][after].prohibited_no_sp = true; + before = LBP_PO; after = LBP_OP; table[before][after].prohibited_no_sp = true; + before = LBP_PO; after = LBP_NU; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_OP; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_NU; table[before][after].prohibited_no_sp = true; + before = LBP_HY; after = LBP_NU; table[before][after].prohibited_no_sp = true; + before = LBP_IS; after = LBP_NU; table[before][after].prohibited_no_sp = true; + before = LBP_NU; after = LBP_NU; table[before][after].prohibited_no_sp = true; + before = LBP_SY; after = LBP_NU; table[before][after].prohibited_no_sp = true; + + /* (LB24) Do not break between numeric prefix/postfix and letters, or between + letters and prefix/postfix. */ + before = LBP_PR; after = LBP_AL; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_HL; table[before][after].prohibited_no_sp = true; + before = LBP_PO; after = LBP_AL; table[before][after].prohibited_no_sp = true; + before = LBP_PO; after = LBP_HL; table[before][after].prohibited_no_sp = true; + before = LBP_AL; after = LBP_PR; table[before][after].prohibited_no_sp = true; + before = LBP_AL; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_HL; after = LBP_PR; table[before][after].prohibited_no_sp = true; + before = LBP_HL; after = LBP_PO; table[before][after].prohibited_no_sp = true; + + /* (LB23a) Do not break between numeric prefixes and ideographs, or between + ideographs and numeric postfixes. */ + before = LBP_PR; after = LBP_ID; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_EB; table[before][after].prohibited_no_sp = true; + before = LBP_PR; after = LBP_EM; table[before][after].prohibited_no_sp = true; + before = LBP_ID; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_EB; after = LBP_PO; table[before][after].prohibited_no_sp = true; + before = LBP_EM; after = LBP_PO; table[before][after].prohibited_no_sp = true; + + /* (LB23) Do not break between digits and letters. */ + before = LBP_AL; after = LBP_NU; table[before][after].prohibited_no_sp = true; + before = LBP_HL; after = LBP_NU; table[before][after].prohibited_no_sp = true; + before = LBP_NU; after = LBP_AL; table[before][after].prohibited_no_sp = true; + before = LBP_NU; after = LBP_HL; table[before][after].prohibited_no_sp = true; + + /* (LB22) Do not break between two ellipses, or between letters, numbers or + exclamations and ellipsis. */ + before = LBP_AL; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_HL; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_EX; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_ID; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_EB; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_EM; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_IN; after = LBP_IN; table[before][after].prohibited_no_sp = true; + before = LBP_NU; after = LBP_IN; table[before][after].prohibited_no_sp = true; + + /* (LB21b) Don’t break between Solidus and Hebrew letters. */ + before = LBP_SY; after = LBP_HL; table[before][after].prohibited_no_sp = true; + + /* (LB21) Do not break before hyphen-minus, other hyphens, fixed-width spaces, + small kana, and other non-starters, or after acute accents. */ + for (before = 0; before < NLBP; before++) + { + after = LBP_BA; table[before][after].prohibited_no_sp = true; + after = LBP_HY; table[before][after].prohibited_no_sp = true; + after = LBP_NS; table[before][after].prohibited_no_sp = true; + } + for (after = 0; after < NLBP; after++) + { + before = LBP_BB; table[before][after].prohibited_no_sp = true; + } + + /* (LB19) Do not break before or after quotation marks, such as '”'. */ + for (before = 0; before < NLBP; before++) + { + after = LBP_QU; table[before][after].prohibited_no_sp = true; + } + for (after = 0; after < NLBP; after++) + { + before = LBP_QU; table[before][after].prohibited_no_sp = true; + } + + /* (LB18) Break after spaces. */ + for (before = 0; before < NLBP; before++) + for (after = 0; after < NLBP; after++) + table[before][after].prohibited_with_sp = false; + + /* (LB17) Do not break within '——', even with intervening spaces. */ + before = LBP_B2; after = LBP_B2; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + + /* (LB16) Do not break between closing punctuation and a nonstarter (lb=NS), + even with intervening spaces. */ + before = LBP_CL; after = LBP_NS; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + before = LBP_CL; after = LBP_CP; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + + /* (LB15) Do not break within '”[', even with intervening spaces. */ + before = LBP_QU; after = LBP_OP; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + + /* (LB14) Do not break after '[', even after spaces. */ + for (after = 0; after < NLBP; after++) + { + before = LBP_OP; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + } + + /* (LB13) Do not break before ‘]’ or ‘!’ or ‘;’ or ‘/’, even after spaces. */ + for (before = 0; before < NLBP; before++) + { + after = LBP_CL; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + after = LBP_CP; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + after = LBP_EX; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + after = LBP_IS; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + after = LBP_SY; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + } + + /* (LB12a) Do not break before NBSP and related characters, except after + spaces and hyphens. */ + for (before = 0; before < NLBP; before++) + if (before != LBP_BA && before != LBP_HY) + { + after = LBP_GL; table[before][after].prohibited_no_sp = true; + } + + /* (LB12) Do not break after NBSP and related characters. */ + for (after = 0; after < NLBP; after++) + { + before = LBP_GL; table[before][after].prohibited_no_sp = true; + } + + /* (LB11) Do not break before or after Word joiner and related characters. */ + for (before = 0; before < NLBP; before++) + { + after = LBP_WJ; table[before][after].prohibited_no_sp = true; + table[before][after].prohibited_with_sp = true; + } + for (after = 0; after < NLBP; after++) + { + before = LBP_WJ; table[before][after].prohibited_no_sp = true; + } + + /* (LB10) Treat any remaining combining mark or ZWJ as AL. */ + /* We resolve LBP_CM at runtime, before accessing the table. */ + for (before = 0; before < NLBP; before++) + table[before][LBP_ZWJ] = table[before][LBP_AL]; + for (after = 0; after < NLBP; after++) + table[LBP_ZWJ][after] = table[LBP_AL][after]; + table[LBP_ZWJ][LBP_ZWJ] = table[LBP_AL][LBP_AL]; + + /* (LB8a) Do not break between a zero width joiner and an ideograph, emoji + base or emoji modifier. */ + before = LBP_ZWJ; after = LBP_ID; table[before][after].prohibited_no_sp = true; + before = LBP_ZWJ; after = LBP_EB; table[before][after].prohibited_no_sp = true; + before = LBP_ZWJ; after = LBP_EM; table[before][after].prohibited_no_sp = true; + + /* Not reflected in the table: + (LB30a) Break between two regional indicator symbols if and only if there are + an even number of regional indicators preceding the position of the + break. + (LB21a) Don't break after Hebrew + Hyphen. + (LB20) Break before and after unresolved CB. + We resolve LBP_CB at runtime, before accessing the table. + (LB9) Do not break a combining character sequence; treat it as if it has the + line breaking class of the base character in all of the following rules. + Treat ZWJ as if it were CM. + (LB8) Break before any character following a zero-width space, even if one + or more spaces intervene. + We handle LBP_ZW at runtime, before accessing the table. + (LB7) Do not break before spaces or zero width space. + We handle LBP_ZW at runtime, before accessing the table. + (LB6) Do not break before hard line breaks. + We handle LBP_BK at runtime, before accessing the table. + (LB5) Treat CR followed by LF, as well as CR, LF, and NL as hard line breaks. + (LB4) Always break after hard line breaks. + (LB3) Always break at the end of text. + (LB2) Never break at the start of text. + */ + + fprintf (stream, "const unsigned char unilbrk_table[%u][%u] =\n", NLBP, NLBP); + fprintf (stream, "{\n"); + fprintf (stream, " /* after */\n"); + + fprintf (stream, " /* "); + for (after = 0; after < NLBP; after++) + fprintf (stream, " %-3s", lbp_value_to_string (after) + 4); + fprintf (stream, " */\n"); + + for (before = 0; before < NLBP; before++) + { + fprintf (stream, "/* %3s */ {", lbp_value_to_string (before) + 4); + for (after = 0; after < NLBP; after++) + { + if (table[before][after].prohibited_no_sp) + { + if (table[before][after].prohibited_with_sp) + /* Prohibited break. */ + fprintf (stream, " P,"); + else + /* Indirect break. */ + fprintf (stream, " I,"); + } + else + { + if (table[before][after].prohibited_with_sp) + abort (); + else + /* Direct break. */ + fprintf (stream, " D,"); + } + } + fprintf (stream, " },\n"); + } + fprintf (stream, "/* \"\" */\n"); + fprintf (stream, "/* before */\n"); + fprintf (stream, "};\n"); + + if (ferror (stream) || fclose (stream)) + { + fprintf (stderr, "error writing to '%s'\n", filename); + exit (1); + } +} + /* ========================================================================= */ /* Word break property. @@ -10582,6 +10929,7 @@ main (int argc, char * argv[]) debug_output_lbrk_tables ("unilbrk/lbrkprop.txt"); debug_output_org_lbrk_tables ("unilbrk/lbrkprop_org.txt"); output_lbrk_tables ("unilbrk/lbrkprop1.h", "unilbrk/lbrkprop2.h", version); + output_lbrk_rules_as_tables ("unilbrk/lbrktables.c", version); debug_output_wbrk_tables ("uniwbrk/wbrkprop.txt"); debug_output_org_wbrk_tables ("uniwbrk/wbrkprop_org.txt"); diff --git a/lib/unilbrk/lbrktables.c b/lib/unilbrk/lbrktables.c index bdfef0067b..dc6b3931cf 100644 --- a/lib/unilbrk/lbrktables.c +++ b/lib/unilbrk/lbrktables.c @@ -1,6 +1,8 @@ -/* Line breaking auxiliary tables. - Copyright (C) 2001-2003, 2006-2021 Free Software Foundation, Inc. - Written by Bruno Haible , 2001. +/* DO NOT EDIT! GENERATED AUTOMATICALLY! */ +/* Table that encodes several line breaking rules. */ +/* Generated automatically by gen-uni-tables.c for Unicode 10.0.0. */ + +/* Copyright (C) 2001-2021 Free Software Foundation, Inc. This file is free software. It is dual-licensed under "the GNU LGPLv3+ or the GNU GPLv2+". @@ -37,41 +39,34 @@ const unsigned char unilbrk_table[30][30] = /* WJ GL B2 BA BB HY CL CP EX IN NS OP QU IS NU PO PR SY AL H2 H3 ID JL JV JT HL RI ZWJ EB EM */ /* WJ */ { P, I, I, I, I, I, P, P, P, I, I, I, I, P, I, I, I, P, I, I, I, I, I, I, I, I, I, I, I, I, }, /* GL */ { P, I, I, I, I, I, P, P, P, I, I, I, I, P, I, I, I, P, I, I, I, I, I, I, I, I, I, I, I, I, }, -/* B2 */ { P, I, P, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, -/* BA */ { P, D, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, +/* B2 */ { P, I, P, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, +/* BA */ { P, D, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, /* BB */ { P, I, I, I, I, I, P, P, P, I, I, I, I, P, I, I, I, P, I, I, I, I, I, I, I, I, I, I, I, I, }, -/* HY */ { P, D, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, -/* CL */ { P, I, D, I, D, I, P, P, P, D, P, D, I, P, D, I, I, P, D, D, D, D, D, D, D, D, D, I, D, D, }, -/* CP */ { P, I, D, I, D, I, P, P, P, D, P, D, I, P, I, I, I, P, I, D, D, D, D, D, D, I, D, I, D, D, }, -/* EX */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, -/* IN */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, -/* NS */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, +/* HY */ { P, D, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, +/* CL */ { P, I, D, I, D, I, P, P, P, D, P, D, I, P, D, I, I, P, D, D, D, D, D, D, D, D, D, D, D, D, }, +/* CP */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, I, I, I, P, I, D, D, D, D, D, D, I, D, I, D, D, }, +/* EX */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, +/* IN */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, +/* NS */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, /* OP */ { P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, }, /* QU */ { P, I, I, I, I, I, P, P, P, I, I, P, I, P, I, I, I, P, I, I, I, I, I, I, I, I, I, I, I, I, }, -/* IS */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, +/* IS */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, I, D, D, D, D, D, D, I, D, I, D, D, }, /* NU */ { P, I, D, I, D, I, P, P, P, I, I, I, I, P, I, I, I, P, I, D, D, D, D, D, D, I, D, I, D, D, }, /* PO */ { P, I, D, I, D, I, P, P, P, D, I, I, I, P, I, D, D, P, I, D, D, D, D, D, D, I, D, I, D, D, }, /* PR */ { P, I, D, I, D, I, P, P, P, D, I, I, I, P, I, D, D, P, I, I, I, I, I, I, I, I, D, I, I, I, }, -/* SY */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, D, D, D, D, D, D, D, I, D, I, D, D, }, -/* AL */ { P, I, D, I, D, I, P, P, P, I, I, I, I, P, I, D, D, P, I, D, D, D, D, D, D, I, D, I, D, D, }, -/* H2 */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, I, I, D, D, I, D, D, }, -/* H3 */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, I, D, D, I, D, D, }, -/* ID */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, D, D, D, I, D, D, }, -/* JL */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, I, I, D, I, I, D, D, D, I, D, D, }, -/* JV */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, I, I, D, D, I, D, D, }, -/* JT */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, I, D, D, I, D, D, }, -/* HL */ { P, I, D, I, D, I, P, P, P, I, I, I, I, P, I, D, D, P, I, D, D, D, D, D, D, I, D, I, D, D, }, -/* RI */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, I, D, I, I, D, D, }, -/* ZWJ */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, I, D, D, I, D, I, I, I, I, }, -/* EB */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, I, D, I, I, D, I, }, -/* EM */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, I, D, I, I, D, D, }, +/* SY */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, I, D, D, P, D, D, D, D, D, D, D, I, D, D, D, D, }, +/* AL */ { P, I, D, I, D, I, P, P, P, I, I, I, I, P, I, I, I, P, I, D, D, D, D, D, D, I, D, I, D, D, }, +/* H2 */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, I, I, D, D, D, D, D, }, +/* H3 */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, I, D, D, D, D, D, }, +/* ID */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, +/* JL */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, I, I, D, I, I, D, D, D, D, D, D, }, +/* JV */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, I, I, D, D, D, D, D, }, +/* JT */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, I, D, D, D, D, D, }, +/* HL */ { P, I, D, I, D, I, P, P, P, I, I, I, I, P, I, I, I, P, I, D, D, D, D, D, D, I, D, I, D, D, }, +/* RI */ { P, I, D, I, D, I, P, P, P, D, I, D, I, P, D, D, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, +/* ZWJ */ { P, I, D, I, D, I, P, P, P, I, I, I, I, P, I, I, I, P, I, D, D, I, D, D, D, I, D, I, I, I, }, +/* EB */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, D, D, D, D, D, I, }, +/* EM */ { P, I, D, I, D, I, P, P, P, I, I, D, I, P, D, I, D, P, D, D, D, D, D, D, D, D, D, D, D, D, }, /* "" */ /* before */ }; -/* Note: The (IS,AL) entry has been changed from I to D. In other words, the - rule "Do not break between numeric punctuation and alphabetics" is not - implemented here. We want to break before the HTML tag in strings like - "

Some sentence.

" */ -/* Note: The (B2,B2) entry should probably be D instead of P. */ -/* Note: The (PR,ID) entry should probably be D instead of I. */ -/* Note: The (WJ,*) and (GL,*) entries should probably be P instead of I. */ diff --git a/tests/unilbrk/test-u16-possible-linebreaks.c b/tests/unilbrk/test-u16-possible-linebreaks.c index 499c56059d..b27d75b2bc 100644 --- a/tests/unilbrk/test-u16-possible-linebreaks.c +++ b/tests/unilbrk/test-u16-possible-linebreaks.c @@ -51,7 +51,7 @@ main () ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : i == 5 || i == 11 || i == 25 - || i == 29 || i == 30 + || i == 29 || i == 45 || i == 51 || i == 52 || i == 53 || i == 55 || i == 56 || i == 58 || i == 59 ? UC_BREAK_POSSIBLE : @@ -70,7 +70,7 @@ main () ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : i == 5 || i == 11 || i == 25 - || i == 29 || i == 30 + || i == 29 || i == 37 || i == 45 || i == 51 || i == 52 || i == 53 || i == 55 || i == 56 || i == 58 || i == 59 ? UC_BREAK_POSSIBLE : diff --git a/tests/unilbrk/test-u32-possible-linebreaks.c b/tests/unilbrk/test-u32-possible-linebreaks.c index aa8708326c..1474ae1c27 100644 --- a/tests/unilbrk/test-u32-possible-linebreaks.c +++ b/tests/unilbrk/test-u32-possible-linebreaks.c @@ -51,7 +51,7 @@ main () ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : i == 5 || i == 11 || i == 25 - || i == 29 || i == 30 + || i == 29 || i == 45 || i == 51 || i == 52 || i == 53 || i == 55 || i == 56 || i == 58 || i == 59 ? UC_BREAK_POSSIBLE : @@ -70,7 +70,7 @@ main () ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : i == 5 || i == 11 || i == 25 - || i == 29 || i == 30 + || i == 29 || i == 37 || i == 45 || i == 51 || i == 52 || i == 53 || i == 55 || i == 56 || i == 58 || i == 59 ? UC_BREAK_POSSIBLE : diff --git a/tests/unilbrk/test-u8-possible-linebreaks.c b/tests/unilbrk/test-u8-possible-linebreaks.c index e23fa33e83..c55e900696 100644 --- a/tests/unilbrk/test-u8-possible-linebreaks.c +++ b/tests/unilbrk/test-u8-possible-linebreaks.c @@ -45,7 +45,7 @@ main () ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY : i == 7 || i == 13 || i == 39 - || i == 43 || i == 44 + || i == 43 || i == 61 || i == 67 || i == 70 || i == 73 || i == 77 || i == 80 || i == 84 || i == 87 ? UC_BREAK_POSSIBLE : @@ -64,7 +64,7 @@ main () ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY : i == 7 || i == 13 || i == 39 - || i == 43 || i == 44 + || i == 43 || i == 52 || i == 61 || i == 67 || i == 70 || i == 73 || i == 77 || i == 80 || i == 84 || i == 87 ? UC_BREAK_POSSIBLE : diff --git a/tests/unilbrk/test-ulc-possible-linebreaks.c b/tests/unilbrk/test-ulc-possible-linebreaks.c index 1fbaafd86a..24bae4b717 100644 --- a/tests/unilbrk/test-ulc-possible-linebreaks.c +++ b/tests/unilbrk/test-ulc-possible-linebreaks.c @@ -42,7 +42,7 @@ main () for (i = 0; i < 36; i++) { ASSERT (p[i] == (i == 35 ? UC_BREAK_MANDATORY : - i == 5 || i == 11 || i == 15 || i == 16 + i == 5 || i == 11 || i == 15 || i == 31 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); } diff --git a/tests/unilbrk/test-ulc-width-linebreaks.c b/tests/unilbrk/test-ulc-width-linebreaks.c index b0a17f687b..78fbe7e5dc 100644 --- a/tests/unilbrk/test-ulc-width-linebreaks.c +++ b/tests/unilbrk/test-ulc-width-linebreaks.c @@ -42,7 +42,7 @@ main () for (i = 0; i < 36; i++) { ASSERT (p[i] == (i == 35 ? UC_BREAK_MANDATORY : - i == 11 || i == 16 || i == 31 ? UC_BREAK_POSSIBLE : + i == 11 || i == 15 || i == 31 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); } free (p); -- 2.39.5