From: Bruno Haible Date: Wed, 29 Dec 2021 18:54:46 +0000 (+0100) Subject: unilbrk: Improve support for strings that contain CR-LF sequences. X-Git-Tag: v1.0~2452 X-Git-Url: https://gitweb.git.savannah.gnu.org/gitweb/?a=commitdiff_plain;h=5209414e95ed830b8b687ae71be8911ef0969e3e;p=gnulib.git unilbrk: Improve support for strings that contain CR-LF sequences. * lib/gen-uni-tables.c (LBP_CR, LBP_LF): New enum values. (get_lbp): For '\r' and '\n', return LBP_CR and LBP_LF, respectively, instead of LBP_BK. (debug_output_lbp, fill_org_lbp, debug_output_org_lbp, lbp_value_to_string): Handle LBP_CR, LBP_LF. * lib/unilbrk/lbrkprop2.h: Regenerated. * lib/unilbrk/lbrktables.h (LBP_CR, LBP_LF): New enum values. * lib/unilbrk.in.h (UC_BREAK_CR_BEFORE_LF): New enum value. (u8_possible_linebreaks, u16_possible_linebreaks, u32_possible_linebreaks, ulc_possible_linebreaks, u8_width_linebreaks, u16_width_linebreaks, u32_width_linebreaks, ulc_width_linebreaks): Declare also a _v2 variant, and use a C macro to activate it by default. * lib/unilbrk/internal.h: New file. * lib/unilbrk/u8-possible-linebreaks.c: Include unilbrk/internal.h. (u8_possible_linebreaks_loop): Renamed from u8_possible_linebreaks. Add 'cr' parameter. (u8_possible_linebreaks, u8_possible_linebreaks_v2): New functions. (main): Update. * lib/unilbrk/u16-possible-linebreaks.c: Include unilbrk/internal.h. (u16_possible_linebreaks_loop): Renamed from u16_possible_linebreaks. Add 'cr' parameter. (u16_possible_linebreaks, u16_possible_linebreaks_v2): New functions. * lib/unilbrk/u32-possible-linebreaks.c: Include unilbrk/internal.h. (u32_possible_linebreaks_loop): Renamed from u32_possible_linebreaks. Add 'cr' parameter. (u32_possible_linebreaks, u32_possible_linebreaks_v2): New functions. * lib/unilbrk/ulc-possible-linebreaks.c: Include unilbrk/internal.h, unilbrk/lbrktables.h. (ulc_possible_linebreaks_internal): Renamed from ulc_possible_linebreaks. Add 'cr' parameter. (ulc_possible_linebreaks, ulc_possible_linebreaks_v2): New functions. (main): Update. * lib/unilbrk/u8-width-linebreaks.c: Include unilbrk/internal.h, unilbrk/lbrktables.h. (u8_width_linebreaks_internal): Renamed from u8_width_linebreaks. Add 'cr' parameter. (u8_width_linebreaks, u8_width_linebreaks_v2): New functions. (main): Update. * lib/unilbrk/u16-width-linebreaks.c: Include unilbrk/internal.h, unilbrk/lbrktables.h. (u16_width_linebreaks_internal): Renamed from u16_width_linebreaks. Add 'cr' parameter. (u16_width_linebreaks, u16_width_linebreaks_v2): New functions. * lib/unilbrk/u32-width-linebreaks.c: Include unilbrk/internal.h, unilbrk/lbrktables.h. (u32_width_linebreaks_internal): Renamed from u32_width_linebreaks. Add 'cr' parameter. (u32_width_linebreaks, u32_width_linebreaks_v2): New functions. * lib/unilbrk/ulc-width-linebreaks.c: Include unilbrk/internal.h, unilbrk/lbrktables.h. (ulc_width_linebreaks_internal): Renamed from ulc_width_linebreaks. Add 'cr' parameter. (ulc_width_linebreaks, ulc_width_linebreaks_v2): New functions. (main): Update. * modules/unilbrk/u8-possible-linebreaks (Files): Add lib/unilbrk/internal.h. * modules/unilbrk/u16-possible-linebreaks (Files): Likewise. * modules/unilbrk/u32-possible-linebreaks (Files): Likewise. * modules/unilbrk/u8-width-linebreaks (Files): Likewise. * tests/unilbrk/test-u8-possible-linebreaks.c (test_function): New function, extracted from main. Add a test of CR-LF handling. (main): Invoke it twice. * tests/unilbrk/test-u16-possible-linebreaks.c (test_function): New function, extracted from main. Add a test of CR-LF handling. (main): Invoke it twice. * tests/unilbrk/test-u32-possible-linebreaks.c (test_function): New function, extracted from main. Add a test of CR-LF handling. (main): Invoke it twice. * tests/unilbrk/test-ulc-possible-linebreaks.c (test_function): New function, extracted from main. (main): Invoke it twice. * tests/unilbrk/test-u8-width-linebreaks.c (test_function): New function, extracted from main. (main): Invoke it twice. * tests/unilbrk/test-u16-width-linebreaks.c (test_function): New function, extracted from main. (main): Invoke it twice. * tests/unilbrk/test-u32-width-linebreaks.c (test_function): New function, extracted from main. (main): Invoke it twice. * tests/unilbrk/test-ulc-width-linebreaks.c (test_function): New function, extracted from main. (main): Invoke it twice. --- diff --git a/ChangeLog b/ChangeLog index 06ace37069..7af2943f84 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,90 @@ +2021-12-29 Bruno Haible + + unilbrk: Improve support for strings that contain CR-LF sequences. + * lib/gen-uni-tables.c (LBP_CR, LBP_LF): New enum values. + (get_lbp): For '\r' and '\n', return LBP_CR and LBP_LF, respectively, + instead of LBP_BK. + (debug_output_lbp, fill_org_lbp, debug_output_org_lbp, + lbp_value_to_string): Handle LBP_CR, LBP_LF. + * lib/unilbrk/lbrkprop2.h: Regenerated. + * lib/unilbrk/lbrktables.h (LBP_CR, LBP_LF): New enum values. + * lib/unilbrk.in.h (UC_BREAK_CR_BEFORE_LF): New enum value. + (u8_possible_linebreaks, u16_possible_linebreaks, + u32_possible_linebreaks, ulc_possible_linebreaks, u8_width_linebreaks, + u16_width_linebreaks, u32_width_linebreaks, ulc_width_linebreaks): + Declare also a _v2 variant, and use a C macro to activate it by default. + * lib/unilbrk/internal.h: New file. + * lib/unilbrk/u8-possible-linebreaks.c: Include unilbrk/internal.h. + (u8_possible_linebreaks_loop): Renamed from u8_possible_linebreaks. Add + 'cr' parameter. + (u8_possible_linebreaks, u8_possible_linebreaks_v2): New functions. + (main): Update. + * lib/unilbrk/u16-possible-linebreaks.c: Include unilbrk/internal.h. + (u16_possible_linebreaks_loop): Renamed from u16_possible_linebreaks. + Add 'cr' parameter. + (u16_possible_linebreaks, u16_possible_linebreaks_v2): New functions. + * lib/unilbrk/u32-possible-linebreaks.c: Include unilbrk/internal.h. + (u32_possible_linebreaks_loop): Renamed from u32_possible_linebreaks. + Add 'cr' parameter. + (u32_possible_linebreaks, u32_possible_linebreaks_v2): New functions. + * lib/unilbrk/ulc-possible-linebreaks.c: Include unilbrk/internal.h, + unilbrk/lbrktables.h. + (ulc_possible_linebreaks_internal): Renamed from + ulc_possible_linebreaks. Add 'cr' parameter. + (ulc_possible_linebreaks, ulc_possible_linebreaks_v2): New functions. + (main): Update. + * lib/unilbrk/u8-width-linebreaks.c: Include unilbrk/internal.h, + unilbrk/lbrktables.h. + (u8_width_linebreaks_internal): Renamed from u8_width_linebreaks. Add + 'cr' parameter. + (u8_width_linebreaks, u8_width_linebreaks_v2): New functions. + (main): Update. + * lib/unilbrk/u16-width-linebreaks.c: Include unilbrk/internal.h, + unilbrk/lbrktables.h. + (u16_width_linebreaks_internal): Renamed from u16_width_linebreaks. Add + 'cr' parameter. + (u16_width_linebreaks, u16_width_linebreaks_v2): New functions. + * lib/unilbrk/u32-width-linebreaks.c: Include unilbrk/internal.h, + unilbrk/lbrktables.h. + (u32_width_linebreaks_internal): Renamed from u32_width_linebreaks. Add + 'cr' parameter. + (u32_width_linebreaks, u32_width_linebreaks_v2): New functions. + * lib/unilbrk/ulc-width-linebreaks.c: Include unilbrk/internal.h, + unilbrk/lbrktables.h. + (ulc_width_linebreaks_internal): Renamed from ulc_width_linebreaks. Add + 'cr' parameter. + (ulc_width_linebreaks, ulc_width_linebreaks_v2): New functions. + (main): Update. + * modules/unilbrk/u8-possible-linebreaks (Files): Add + lib/unilbrk/internal.h. + * modules/unilbrk/u16-possible-linebreaks (Files): Likewise. + * modules/unilbrk/u32-possible-linebreaks (Files): Likewise. + * modules/unilbrk/u8-width-linebreaks (Files): Likewise. + * tests/unilbrk/test-u8-possible-linebreaks.c (test_function): New + function, extracted from main. Add a test of CR-LF handling. + (main): Invoke it twice. + * tests/unilbrk/test-u16-possible-linebreaks.c (test_function): New + function, extracted from main. Add a test of CR-LF handling. + (main): Invoke it twice. + * tests/unilbrk/test-u32-possible-linebreaks.c (test_function): New + function, extracted from main. Add a test of CR-LF handling. + (main): Invoke it twice. + * tests/unilbrk/test-ulc-possible-linebreaks.c (test_function): New + function, extracted from main. + (main): Invoke it twice. + * tests/unilbrk/test-u8-width-linebreaks.c (test_function): New + function, extracted from main. + (main): Invoke it twice. + * tests/unilbrk/test-u16-width-linebreaks.c (test_function): New + function, extracted from main. + (main): Invoke it twice. + * tests/unilbrk/test-u32-width-linebreaks.c (test_function): New + function, extracted from main. + (main): Invoke it twice. + * tests/unilbrk/test-ulc-width-linebreaks.c (test_function): New + function, extracted from main. + (main): Invoke it twice. + 2021-12-29 Bruno Haible uniwbrk: Add unit tests of CR-LF sequences. diff --git a/lib/gen-uni-tables.c b/lib/gen-uni-tables.c index 3d1b718067..a5464b225e 100644 --- a/lib/gen-uni-tables.c +++ b/lib/gen-uni-tables.c @@ -6518,20 +6518,20 @@ enum { /* Values >= 30 are resolved at run time. */ LBP_BK = 30, /* mandatory break */ -/*LBP_CR, carriage return - not used here because it's a DOSism */ -/*LBP_LF, line feed - not used here because it's a DOSism */ - LBP_CM = 31, /* attached characters and combining marks */ + LBP_CR = 31, /* carriage return */ + LBP_LF = 32, /* line feed */ + LBP_CM = 33, /* attached characters and combining marks */ /*LBP_NL, next line - not used here because it's equivalent to LBP_BK */ /*LBP_SG, surrogates - not used here because they are not characters */ LBP_WJ = 0, /* word joiner */ - LBP_ZW = 32, /* zero width space */ + LBP_ZW = 34, /* zero width space */ LBP_GL = 1, /* non-breaking (glue) */ - LBP_SP = 33, /* space */ + LBP_SP = 35, /* space */ LBP_B2 = 2, /* break opportunity before and after */ LBP_BA = 3, /* break opportunity after */ LBP_BB = 4, /* break opportunity before */ LBP_HY = 5, /* hyphen */ - LBP_CB = 34, /* contingent break opportunity */ + LBP_CB = 36, /* contingent break opportunity */ LBP_CL = 6, /* closing punctuation */ LBP_CP = 7, /* closing parenthesis */ LBP_EX = 8, /* exclamation/interrogation */ @@ -6544,7 +6544,7 @@ enum LBP_PO = 15, /* postfix (numeric) */ LBP_PR = 16, /* prefix (numeric) */ LBP_SY = 17, /* symbols allowing breaks */ - LBP_AI = 35, /* ambiguous (alphabetic or ideograph) */ + LBP_AI = 37, /* ambiguous (alphabetic or ideograph) */ LBP_AL = 18, /* ordinary alphabetic and symbol characters */ /*LBP_CJ, conditional Japanese starter, resolved to NS */ LBP_H2 = 19, /* Hangul LV syllable */ @@ -6555,11 +6555,11 @@ enum LBP_JV = 23, /* Hangul V Jamo */ LBP_JT = 24, /* Hangul T Jamo */ LBP_RI = 26, /* regional indicator */ - LBP_SA = 36, /* complex context (South East Asian) */ + LBP_SA = 38, /* complex context (South East Asian) */ LBP_ZWJ = 27, /* zero width joiner */ LBP_EB = 28, /* emoji base */ LBP_EM = 29, /* emoji modifier */ - LBP_XX = 37 /* unknown */ + LBP_XX = 39 /* unknown */ }; /* Returns the line breaking classification for ch, as a bit mask. */ @@ -6575,7 +6575,11 @@ get_lbp (unsigned int ch) if (unicode_attributes[ch].name != NULL) { /* mandatory break */ - if (ch == 0x000A || ch == 0x000D || ch == 0x0085 /* newline */ + if (ch == 0x000A) + attr |= (int64_t) 1 << LBP_LF; + if (ch == 0x000D) + attr |= (int64_t) 1 << LBP_CR; + if (ch == 0x0085 /* newline */ || ch == 0x000C /* form feed */ || ch == 0x000B /* line tabulation */ || ch == 0x2028 /* LINE SEPARATOR */ @@ -7206,7 +7210,7 @@ get_lbp (unsigned int ch) && ch != 0x110BD /* KAITHI NUMBER SIGN */ && ch != 0x08E2 /* ARABIC DISPUTED END OF AYAH */) || ch == 0x3035 /* VERTICAL KANA REPEAT MARK LOWER HALF */) - if (!(attr & (((int64_t) 1 << LBP_BK) | ((int64_t) 1 << LBP_BA) | ((int64_t) 1 << LBP_GL) | ((int64_t) 1 << LBP_SA) | ((int64_t) 1 << LBP_WJ) | ((int64_t) 1 << LBP_ZW) | ((int64_t) 1 << LBP_ZWJ)))) + if (!(attr & (((int64_t) 1 << LBP_BK) | ((int64_t) 1 << LBP_CR) | ((int64_t) 1 << LBP_LF) | ((int64_t) 1 << LBP_BA) | ((int64_t) 1 << LBP_GL) | ((int64_t) 1 << LBP_SA) | ((int64_t) 1 << LBP_WJ) | ((int64_t) 1 << LBP_ZW) | ((int64_t) 1 << LBP_ZWJ)))) attr |= (int64_t) 1 << LBP_CM; /* ideographic */ @@ -7587,6 +7591,8 @@ debug_output_lbp (FILE *stream) #define PRINT_BIT(attr,bit) \ if (attr & ((int64_t) 1 << bit)) fprintf (stream, " " #bit); PRINT_BIT(attr,LBP_BK); + PRINT_BIT(attr,LBP_CR); + PRINT_BIT(attr,LBP_LF); PRINT_BIT(attr,LBP_CM); PRINT_BIT(attr,LBP_WJ); PRINT_BIT(attr,LBP_ZW); @@ -7706,6 +7712,8 @@ fill_org_lbp (const char *linebreak_filename) #define TRY(bit) else if (strcmp (field1, #bit + 4) == 0) value = bit; if (false) {} TRY(LBP_BK) + TRY(LBP_CR) + TRY(LBP_LF) TRY(LBP_CM) TRY(LBP_WJ) TRY(LBP_ZW) @@ -7744,8 +7752,6 @@ fill_org_lbp (const char *linebreak_filename) TRY(LBP_EM) TRY(LBP_XX) #undef TRY - else if (strcmp (field1, "LF") == 0) value = LBP_BK; - else if (strcmp (field1, "CR") == 0) value = LBP_BK; else if (strcmp (field1, "NL") == 0) value = LBP_BK; else if (strcmp (field1, "SG") == 0) value = LBP_XX; else if (strcmp (field1, "CJ") == 0) value = LBP_NS; @@ -7792,6 +7798,8 @@ debug_output_org_lbp (FILE *stream) #define PRINT_BIT(attr,bit) \ if (attr == bit) fprintf (stream, " " #bit); PRINT_BIT(attr,LBP_BK); + PRINT_BIT(attr,LBP_CR); + PRINT_BIT(attr,LBP_LF); PRINT_BIT(attr,LBP_CM); PRINT_BIT(attr,LBP_WJ); PRINT_BIT(attr,LBP_ZW); @@ -7865,6 +7873,8 @@ lbp_value_to_string (unsigned int value) { #define CASE(x) case x: value_string = #x; break; CASE(LBP_BK); + CASE(LBP_CR); + CASE(LBP_LF); CASE(LBP_CM); CASE(LBP_WJ); CASE(LBP_ZW); diff --git a/lib/unilbrk.in.h b/lib/unilbrk.in.h index dcf7094a0b..cbf525f81a 100644 --- a/lib/unilbrk.in.h +++ b/lib/unilbrk.in.h @@ -52,11 +52,14 @@ enum UC_BREAK_PROHIBITED, UC_BREAK_POSSIBLE, UC_BREAK_MANDATORY, - UC_BREAK_HYPHENATION + UC_BREAK_HYPHENATION, + UC_BREAK_CR_BEFORE_LF /* only used in _v2 or later */ }; /* Determine the line break points in S, and store the result at p[0..n-1]. p[i] = UC_BREAK_MANDATORY means that s[i] is a line break character. + p[i] = UC_BREAK_CR_BEFORE_LF means that s[i] and s[i+1] is the CR-LF + character sequence. (Only used in _v2 or later.) p[i] = UC_BREAK_POSSIBLE means that a line break may be inserted between s[i-1] and s[i]. p[i] = UC_BREAK_HYPHENATION means that a hyphen and a line break may be @@ -67,15 +70,34 @@ enum extern void u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char *_UC_RESTRICT p); +extern void + u8_possible_linebreaks_v2 (const uint8_t *s, size_t n, + const char *encoding, char *_UC_RESTRICT p); +#define u8_possible_linebreaks u8_possible_linebreaks_v2 + extern void u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char *_UC_RESTRICT p); +extern void + u16_possible_linebreaks_v2 (const uint16_t *s, size_t n, + const char *encoding, char *_UC_RESTRICT p); +#define u16_possible_linebreaks u16_possible_linebreaks_v2 + extern void u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char *_UC_RESTRICT p); +extern void + u32_possible_linebreaks_v2 (const uint32_t *s, size_t n, + const char *encoding, char *_UC_RESTRICT p); +#define u32_possible_linebreaks u32_possible_linebreaks_v2 + extern void ulc_possible_linebreaks (const char *s, size_t n, const char *encoding, char *_UC_RESTRICT p); +extern void + ulc_possible_linebreaks_v2 (const char *s, size_t n, + const char *encoding, char *_UC_RESTRICT p); +#define ulc_possible_linebreaks ulc_possible_linebreaks_v2 /* Choose the best line breaks, assuming the uc_width function. The string is s[0..n-1]. The maximum number of columns per line is given @@ -93,21 +115,48 @@ extern int int start_column, int at_end_columns, const char *o, const char *encoding, char *_UC_RESTRICT p); +extern int + u8_width_linebreaks_v2 (const uint8_t *s, size_t n, int width, + int start_column, int at_end_columns, + const char *o, const char *encoding, + char *_UC_RESTRICT p); +#define u8_width_linebreaks u8_width_linebreaks_v2 + extern int u16_width_linebreaks (const uint16_t *s, size_t n, int width, int start_column, int at_end_columns, const char *o, const char *encoding, char *_UC_RESTRICT p); +extern int + u16_width_linebreaks_v2 (const uint16_t *s, size_t n, int width, + int start_column, int at_end_columns, + const char *o, const char *encoding, + char *_UC_RESTRICT p); +#define u16_width_linebreaks u16_width_linebreaks_v2 + extern int u32_width_linebreaks (const uint32_t *s, size_t n, int width, int start_column, int at_end_columns, const char *o, const char *encoding, char *_UC_RESTRICT p); +extern int + u32_width_linebreaks_v2 (const uint32_t *s, size_t n, int width, + int start_column, int at_end_columns, + const char *o, const char *encoding, + char *_UC_RESTRICT p); +#define u32_width_linebreaks u32_width_linebreaks_v2 + extern int ulc_width_linebreaks (const char *s, size_t n, int width, int start_column, int at_end_columns, const char *o, const char *encoding, char *_UC_RESTRICT p); +extern int + ulc_width_linebreaks_v2 (const char *s, size_t n, int width, + int start_column, int at_end_columns, + const char *o, const char *encoding, + char *_UC_RESTRICT p); +#define ulc_width_linebreaks ulc_width_linebreaks_v2 #ifdef __cplusplus diff --git a/lib/unilbrk/internal.h b/lib/unilbrk/internal.h new file mode 100644 index 0000000000..e1e694bbb8 --- /dev/null +++ b/lib/unilbrk/internal.h @@ -0,0 +1,43 @@ +/* Internal functions for line breaking of Unicode strings. + Copyright (C) 2001-2003, 2005-2021 Free Software Foundation, Inc. + Written by Bruno Haible , 2021. + + This file is free software. + It is dual-licensed under "the GNU LGPLv3+ or the GNU GPLv2+". + You can redistribute it and/or modify it under either + - the terms of the GNU Lesser General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version, or + - the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2, or (at your option) + any later version, or + - the same dual license "the GNU LGPLv3+ or the GNU GPLv2+". + + This file is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License and the GNU General Public License + for more details. + + You should have received a copy of the GNU Lesser General Public + License and of the GNU General Public License along with this + program. If not, see . */ + +extern void + u8_possible_linebreaks_loop (const uint8_t *s, size_t n, + const char *encoding, int cr, + char *_UC_RESTRICT p); +extern void + u16_possible_linebreaks_loop (const uint16_t *s, size_t n, + const char *encoding, int cr, + char *_UC_RESTRICT p); +extern void + u32_possible_linebreaks_loop (const uint32_t *s, size_t n, + const char *encoding, int cr, + char *_UC_RESTRICT p); + +extern int + u8_width_linebreaks_internal (const uint8_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, int cr, + char *p); diff --git a/lib/unilbrk/lbrkprop2.h b/lib/unilbrk/lbrkprop2.h index d05611555b..f1ea56fcf1 100644 --- a/lib/unilbrk/lbrkprop2.h +++ b/lib/unilbrk/lbrkprop2.h @@ -291,7 +291,7 @@ const lbrkprop_t unilbrkprop = }, { LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, - LBP_CM, LBP_BA, LBP_BK, LBP_BK, LBP_BK, LBP_BK, LBP_CM, LBP_CM, + LBP_CM, LBP_BA, LBP_LF, LBP_BK, LBP_BK, LBP_CR, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_SP, LBP_EX, LBP_QU, LBP_AL, LBP_PR, LBP_PO, LBP_AL, LBP_QU, diff --git a/lib/unilbrk/lbrktables.h b/lib/unilbrk/lbrktables.h index 31ce91ac17..da5dc7eac7 100644 --- a/lib/unilbrk/lbrktables.h +++ b/lib/unilbrk/lbrktables.h @@ -31,20 +31,20 @@ enum { /* Values >= 30 are resolved at run time. */ LBP_BK = 30, /* mandatory break */ -/*LBP_CR, carriage return - not used here because it's a DOSism */ -/*LBP_LF, line feed - not used here because it's a DOSism */ - LBP_CM = 31, /* attached characters and combining marks */ + LBP_CR = 31, /* carriage return */ + LBP_LF = 32, /* line feed */ + LBP_CM = 33, /* attached characters and combining marks */ /*LBP_NL, next line - not used here because it's equivalent to LBP_BK */ /*LBP_SG, surrogates - not used here because they are not characters */ LBP_WJ = 0, /* word joiner */ - LBP_ZW = 32, /* zero width space */ + LBP_ZW = 34, /* zero width space */ LBP_GL = 1, /* non-breaking (glue) */ - LBP_SP = 33, /* space */ + LBP_SP = 35, /* space */ LBP_B2 = 2, /* break opportunity before and after */ LBP_BA = 3, /* break opportunity after */ LBP_BB = 4, /* break opportunity before */ LBP_HY = 5, /* hyphen */ - LBP_CB = 34, /* contingent break opportunity */ + LBP_CB = 36, /* contingent break opportunity */ LBP_CL = 6, /* closing punctuation */ LBP_CP = 7, /* closing parenthesis */ LBP_EX = 8, /* exclamation/interrogation */ @@ -57,7 +57,7 @@ enum LBP_PO = 15, /* postfix (numeric) */ LBP_PR = 16, /* prefix (numeric) */ LBP_SY = 17, /* symbols allowing breaks */ - LBP_AI = 35, /* ambiguous (alphabetic or ideograph) */ + LBP_AI = 37, /* ambiguous (alphabetic or ideograph) */ LBP_AL = 18, /* ordinary alphabetic and symbol characters */ /*LBP_CJ, conditional Japanese starters, resolved to NS */ LBP_H2 = 19, /* Hangul LV syllable */ @@ -68,11 +68,11 @@ enum LBP_JV = 23, /* Hangul V Jamo */ LBP_JT = 24, /* Hangul T Jamo */ LBP_RI = 26, /* regional indicator */ - LBP_SA = 36, /* complex context (South East Asian) */ + LBP_SA = 38, /* complex context (South East Asian) */ LBP_ZWJ = 27, /* zero width joiner */ LBP_EB = 28, /* emoji base */ LBP_EM = 29, /* emoji modifier */ - LBP_XX = 37, /* unknown */ + LBP_XX = 39, /* unknown */ /* Artificial values that exist only at runtime, not in the tables. */ LBP_HL_BA = 100 }; diff --git a/lib/unilbrk/u16-possible-linebreaks.c b/lib/unilbrk/u16-possible-linebreaks.c index ee11730450..087c7b3b26 100644 --- a/lib/unilbrk/u16-possible-linebreaks.c +++ b/lib/unilbrk/u16-possible-linebreaks.c @@ -27,6 +27,7 @@ /* Specification. */ #include "unilbrk.h" +#include "unilbrk/internal.h" #include #include @@ -36,7 +37,8 @@ #include "unistr.h" void -u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char *p) +u16_possible_linebreaks_loop (const uint16_t *s, size_t n, const char *encoding, + int cr, char *p) { if (n > 0) { @@ -59,11 +61,15 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char int count = u16_mbtouc_unsafe (&uc, s, s_end - s); int prop = unilbrkprop_lookup (uc); - if (prop == LBP_BK) + if (prop == LBP_BK || prop == LBP_LF || prop == LBP_CR) { /* (LB4,LB5,LB6) Mandatory break. */ *p = UC_BREAK_MANDATORY; - prev_prop = LBP_BK; + /* cr is either LBP_CR or -1. In the first case, recognize + a CR-LF sequence. */ + if (prev_prop == cr && prop == LBP_LF) + p[-1] = UC_BREAK_CR_BEFORE_LF; + prev_prop = prop; last_prop = LBP_BK; seen_space = NULL; } @@ -207,3 +213,19 @@ u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char while (s < s_end); } } + +#undef u16_possible_linebreaks + +void +u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, + char *p) +{ + u16_possible_linebreaks_loop (s, n, encoding, -1, p); +} + +void +u16_possible_linebreaks_v2 (const uint16_t *s, size_t n, const char *encoding, + char *p) +{ + u16_possible_linebreaks_loop (s, n, encoding, LBP_CR, p); +} diff --git a/lib/unilbrk/u16-width-linebreaks.c b/lib/unilbrk/u16-width-linebreaks.c index 6de573560c..bcb967c57c 100644 --- a/lib/unilbrk/u16-width-linebreaks.c +++ b/lib/unilbrk/u16-width-linebreaks.c @@ -28,21 +28,23 @@ /* Specification. */ #include "unilbrk.h" +#include "unilbrk/internal.h" +#include "unilbrk/lbrktables.h" #include "unistr.h" #include "uniwidth.h" -int -u16_width_linebreaks (const uint16_t *s, size_t n, - int width, int start_column, int at_end_columns, - const char *o, const char *encoding, - char *p) +static int +u16_width_linebreaks_internal (const uint16_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, int cr, + char *p) { const uint16_t *s_end; char *last_p; int last_column; int piece_width; - u16_possible_linebreaks (s, n, encoding, p); + u16_possible_linebreaks_loop (s, n, encoding, cr, p); s_end = s + n; last_p = NULL; @@ -57,7 +59,8 @@ u16_width_linebreaks (const uint16_t *s, size_t n, if (o != NULL && *o != UC_BREAK_UNDEFINED) *p = *o; - if (*p == UC_BREAK_POSSIBLE || *p == UC_BREAK_MANDATORY) + if (*p == UC_BREAK_POSSIBLE + || *p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF) { /* An atomic piece of text ends here. */ if (last_p != NULL && last_column + piece_width > width) @@ -68,7 +71,7 @@ u16_width_linebreaks (const uint16_t *s, size_t n, } } - if (*p == UC_BREAK_MANDATORY) + if (*p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF) { /* uc is a line break character. */ /* Start a new piece at column 0. */ @@ -114,3 +117,27 @@ u16_width_linebreaks (const uint16_t *s, size_t n, return last_column + piece_width; } + +#undef u16_width_linebreaks + +int +u16_width_linebreaks (const uint16_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return u16_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, -1, p); +} + +int +u16_width_linebreaks_v2 (const uint16_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return u16_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, LBP_CR, p); +} diff --git a/lib/unilbrk/u32-possible-linebreaks.c b/lib/unilbrk/u32-possible-linebreaks.c index d36e2db882..3cecb11d04 100644 --- a/lib/unilbrk/u32-possible-linebreaks.c +++ b/lib/unilbrk/u32-possible-linebreaks.c @@ -27,6 +27,7 @@ /* Specification. */ #include "unilbrk.h" +#include "unilbrk/internal.h" #include @@ -34,7 +35,8 @@ #include "uniwidth/cjk.h" void -u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char *p) +u32_possible_linebreaks_loop (const uint32_t *s, size_t n, const char *encoding, + int cr, char *p) { if (n > 0) { @@ -53,11 +55,15 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char ucs4_t uc = *s; int prop = unilbrkprop_lookup (uc); - if (prop == LBP_BK) + if (prop == LBP_BK || prop == LBP_LF || prop == LBP_CR) { /* (LB4,LB5,LB6) Mandatory break. */ *p = UC_BREAK_MANDATORY; - prev_prop = LBP_BK; + /* cr is either LBP_CR or -1. In the first case, recognize + a CR-LF sequence. */ + if (prev_prop == cr && prop == LBP_LF) + p[-1] = UC_BREAK_CR_BEFORE_LF; + prev_prop = prop; last_prop = LBP_BK; seen_space = NULL; } @@ -201,3 +207,19 @@ u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char while (s < s_end); } } + +#undef u32_possible_linebreaks + +void +u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, + char *p) +{ + u32_possible_linebreaks_loop (s, n, encoding, -1, p); +} + +void +u32_possible_linebreaks_v2 (const uint32_t *s, size_t n, const char *encoding, + char *p) +{ + u32_possible_linebreaks_loop (s, n, encoding, LBP_CR, p); +} diff --git a/lib/unilbrk/u32-width-linebreaks.c b/lib/unilbrk/u32-width-linebreaks.c index 00f1a90ef7..a4c3ead52c 100644 --- a/lib/unilbrk/u32-width-linebreaks.c +++ b/lib/unilbrk/u32-width-linebreaks.c @@ -28,20 +28,22 @@ /* Specification. */ #include "unilbrk.h" +#include "unilbrk/internal.h" +#include "unilbrk/lbrktables.h" #include "uniwidth.h" -int -u32_width_linebreaks (const uint32_t *s, size_t n, - int width, int start_column, int at_end_columns, - const char *o, const char *encoding, - char *p) +static int +u32_width_linebreaks_internal (const uint32_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, int cr, + char *p) { const uint32_t *s_end; char *last_p; int last_column; int piece_width; - u32_possible_linebreaks (s, n, encoding, p); + u32_possible_linebreaks_loop (s, n, encoding, cr, p); s_end = s + n; last_p = NULL; @@ -55,7 +57,8 @@ u32_width_linebreaks (const uint32_t *s, size_t n, if (o != NULL && *o != UC_BREAK_UNDEFINED) *p = *o; - if (*p == UC_BREAK_POSSIBLE || *p == UC_BREAK_MANDATORY) + if (*p == UC_BREAK_POSSIBLE + || *p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF) { /* An atomic piece of text ends here. */ if (last_p != NULL && last_column + piece_width > width) @@ -66,7 +69,7 @@ u32_width_linebreaks (const uint32_t *s, size_t n, } } - if (*p == UC_BREAK_MANDATORY) + if (*p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF) { /* uc is a line break character. */ /* Start a new piece at column 0. */ @@ -112,3 +115,27 @@ u32_width_linebreaks (const uint32_t *s, size_t n, return last_column + piece_width; } + +#undef u32_width_linebreaks + +int +u32_width_linebreaks (const uint32_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return u32_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, -1, p); +} + +int +u32_width_linebreaks_v2 (const uint32_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return u32_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, LBP_CR, p); +} diff --git a/lib/unilbrk/u8-possible-linebreaks.c b/lib/unilbrk/u8-possible-linebreaks.c index 06b914af91..c585408815 100644 --- a/lib/unilbrk/u8-possible-linebreaks.c +++ b/lib/unilbrk/u8-possible-linebreaks.c @@ -27,6 +27,7 @@ /* Specification. */ #include "unilbrk.h" +#include "unilbrk/internal.h" #include #include @@ -36,7 +37,8 @@ #include "unistr.h" void -u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char *p) +u8_possible_linebreaks_loop (const uint8_t *s, size_t n, const char *encoding, + int cr, char *p) { if (n > 0) { @@ -59,11 +61,15 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * int count = u8_mbtouc_unsafe (&uc, s, s_end - s); int prop = unilbrkprop_lookup (uc); - if (prop == LBP_BK) + if (prop == LBP_BK || prop == LBP_LF || prop == LBP_CR) { /* (LB4,LB5,LB6) Mandatory break. */ *p = UC_BREAK_MANDATORY; - prev_prop = LBP_BK; + /* cr is either LBP_CR or -1. In the first case, recognize + a CR-LF sequence. */ + if (prev_prop == cr && prop == LBP_LF) + p[-1] = UC_BREAK_CR_BEFORE_LF; + prev_prop = prop; last_prop = LBP_BK; seen_space = NULL; } @@ -208,6 +214,22 @@ u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char * } } +#undef u8_possible_linebreaks + +void +u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, + char *p) +{ + u8_possible_linebreaks_loop (s, n, encoding, -1, p); +} + +void +u8_possible_linebreaks_v2 (const uint8_t *s, size_t n, const char *encoding, + char *p) +{ + u8_possible_linebreaks_loop (s, n, encoding, LBP_CR, p); +} + #ifdef TEST @@ -273,7 +295,7 @@ main (int argc, char * argv[]) char *breaks = malloc (length); int i; - u8_possible_linebreaks ((uint8_t *) input, length, "UTF-8", breaks); + u8_possible_linebreaks_v2 ((uint8_t *) input, length, "UTF-8", breaks); for (i = 0; i < length; i++) { @@ -287,6 +309,10 @@ main (int argc, char * argv[]) /* U+21B2 (or U+21B5) in UTF-8 encoding */ putc (0xe2, stdout); putc (0x86, stdout); putc (0xb2, stdout); break; + case UC_BREAK_CR_BEFORE_LF: + /* U+21E4 in UTF-8 encoding */ + putc (0xe2, stdout); putc (0x87, stdout); putc (0xa4, stdout); + break; case UC_BREAK_PROHIBITED: break; default: diff --git a/lib/unilbrk/u8-width-linebreaks.c b/lib/unilbrk/u8-width-linebreaks.c index 7af2706ec5..98a70f066d 100644 --- a/lib/unilbrk/u8-width-linebreaks.c +++ b/lib/unilbrk/u8-width-linebreaks.c @@ -27,22 +27,24 @@ /* Specification. */ #include "unilbrk.h" +#include "unilbrk/internal.h" +#include "unilbrk/lbrktables.h" #include "unistr.h" #include "uniwidth.h" int -u8_width_linebreaks (const uint8_t *s, size_t n, - int width, int start_column, int at_end_columns, - const char *o, const char *encoding, - char *p) +u8_width_linebreaks_internal (const uint8_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, int cr, + char *p) { const uint8_t *s_end; char *last_p; int last_column; int piece_width; - u8_possible_linebreaks (s, n, encoding, p); + u8_possible_linebreaks_loop (s, n, encoding, cr, p); s_end = s + n; last_p = NULL; @@ -57,7 +59,8 @@ u8_width_linebreaks (const uint8_t *s, size_t n, if (o != NULL && *o != UC_BREAK_UNDEFINED) *p = *o; - if (*p == UC_BREAK_POSSIBLE || *p == UC_BREAK_MANDATORY) + if (*p == UC_BREAK_POSSIBLE + || *p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF) { /* An atomic piece of text ends here. */ if (last_p != NULL && last_column + piece_width > width) @@ -68,7 +71,7 @@ u8_width_linebreaks (const uint8_t *s, size_t n, } } - if (*p == UC_BREAK_MANDATORY) + if (*p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF) { /* uc is a line break character. */ /* Start a new piece at column 0. */ @@ -115,6 +118,30 @@ u8_width_linebreaks (const uint8_t *s, size_t n, return last_column + piece_width; } +#undef u8_width_linebreaks + +int +u8_width_linebreaks (const uint8_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return u8_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, -1, p); +} + +int +u8_width_linebreaks_v2 (const uint8_t *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return u8_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, LBP_CR, p); +} + #ifdef TEST @@ -182,7 +209,7 @@ main (int argc, char * argv[]) char *breaks = malloc (length); int i; - u8_width_linebreaks ((uint8_t *) input, length, width, 0, 0, NULL, "UTF-8", breaks); + u8_width_linebreaks_v2 ((uint8_t *) input, length, width, 0, 0, NULL, "UTF-8", breaks); for (i = 0; i < length; i++) { @@ -193,6 +220,8 @@ main (int argc, char * argv[]) break; case UC_BREAK_MANDATORY: break; + case UC_BREAK_CR_BEFORE_LF: + break; case UC_BREAK_PROHIBITED: break; default: diff --git a/lib/unilbrk/ulc-possible-linebreaks.c b/lib/unilbrk/ulc-possible-linebreaks.c index 5d1c30b929..ba2c78fa82 100644 --- a/lib/unilbrk/ulc-possible-linebreaks.c +++ b/lib/unilbrk/ulc-possible-linebreaks.c @@ -33,6 +33,8 @@ #include "c-ctype.h" #include "uniconv.h" +#include "unilbrk/internal.h" +#include "unilbrk/lbrktables.h" #include "unilbrk/ulc-common.h" /* Line breaking of a string in an arbitrary encoding. @@ -48,14 +50,14 @@ but this is not backed by an RFC. So we use UTF-8. It supports characters up to \U7FFFFFFF and is unambiguously defined. */ -void -ulc_possible_linebreaks (const char *s, size_t n, const char *encoding, - char *p) +static void +ulc_possible_linebreaks_internal (const char *s, size_t n, const char *encoding, + int cr, char *p) { if (n > 0) { if (is_utf8_encoding (encoding)) - u8_possible_linebreaks ((const uint8_t *) s, n, encoding, p); + u8_possible_linebreaks_loop ((const uint8_t *) s, n, encoding, cr, p); else { /* Convert the string to UTF-8 and build a translation table @@ -79,7 +81,7 @@ ulc_possible_linebreaks (const char *s, size_t n, const char *encoding, /* Determine the possible line breaks of the UTF-8 string. */ - u8_possible_linebreaks (t, m, encoding, q); + u8_possible_linebreaks_loop (t, m, encoding, cr, q); /* Translate the result back to the original string. */ memset (p, UC_BREAK_PROHIBITED, n); @@ -102,7 +104,7 @@ ulc_possible_linebreaks (const char *s, size_t n, const char *encoding, if (is_all_ascii (s, n)) { /* ASCII is a subset of UTF-8. */ - u8_possible_linebreaks ((const uint8_t *) s, n, encoding, p); + u8_possible_linebreaks_loop ((const uint8_t *) s, n, encoding, cr, p); return; } #endif @@ -114,7 +116,14 @@ ulc_possible_linebreaks (const char *s, size_t n, const char *encoding, const char *s_end = s + n; while (s < s_end) { - *p = (*s == '\n' ? UC_BREAK_MANDATORY : UC_BREAK_PROHIBITED); + *p = (*s == '\n' + ? UC_BREAK_MANDATORY + : ((cr >= 0 + && *s == '\r' + && s + 1 < s_end + && *(s + 1) == '\n') + ? UC_BREAK_CR_BEFORE_LF + : UC_BREAK_PROHIBITED)); s++; p++; } @@ -123,6 +132,22 @@ ulc_possible_linebreaks (const char *s, size_t n, const char *encoding, } } +#undef ulc_possible_linebreaks + +void +ulc_possible_linebreaks (const char *s, size_t n, const char *encoding, + char *p) +{ + ulc_possible_linebreaks_internal (s, n, encoding, -1, p); +} + +void +ulc_possible_linebreaks_v2 (const char *s, size_t n, const char *encoding, + char *p) +{ + ulc_possible_linebreaks_internal (s, n, encoding, LBP_CR, p); +} + #ifdef TEST @@ -190,7 +215,7 @@ main (int argc, char * argv[]) char *breaks = malloc (length); int i; - ulc_possible_linebreaks (input, length, locale_charset (), breaks); + ulc_possible_linebreaks_v2 (input, length, locale_charset (), breaks); for (i = 0; i < length; i++) { @@ -201,6 +226,8 @@ main (int argc, char * argv[]) break; case UC_BREAK_MANDATORY: break; + case UC_BREAK_CR_BEFORE_LF: + break; case UC_BREAK_PROHIBITED: break; default: diff --git a/lib/unilbrk/ulc-width-linebreaks.c b/lib/unilbrk/ulc-width-linebreaks.c index 76a1fe92ff..fcb4b4573c 100644 --- a/lib/unilbrk/ulc-width-linebreaks.c +++ b/lib/unilbrk/ulc-width-linebreaks.c @@ -33,6 +33,8 @@ #include "c-ctype.h" #include "uniconv.h" +#include "unilbrk/internal.h" +#include "unilbrk/lbrktables.h" #include "unilbrk/ulc-common.h" /* Line breaking of a string in an arbitrary encoding. @@ -48,16 +50,16 @@ but this is not backed by an RFC. So we use UTF-8. It supports characters up to \U7FFFFFFF and is unambiguously defined. */ -int -ulc_width_linebreaks (const char *s, size_t n, - int width, int start_column, int at_end_columns, - const char *o, const char *encoding, - char *p) +static int +ulc_width_linebreaks_internal (const char *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, int cr, + char *p) { if (n > 0) { if (is_utf8_encoding (encoding)) - return u8_width_linebreaks ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, p); + return u8_width_linebreaks_internal ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, cr, p); else { /* Convert the string to UTF-8 and build a translation table @@ -94,7 +96,7 @@ ulc_width_linebreaks (const char *s, size_t n, /* Determine the line breaks of the UTF-8 string. */ res_column = - u8_width_linebreaks (t, m, width, start_column, at_end_columns, o8, encoding, q); + u8_width_linebreaks_internal (t, m, width, start_column, at_end_columns, o8, encoding, cr, q); /* Translate the result back to the original string. */ memset (p, UC_BREAK_PROHIBITED, n); @@ -116,7 +118,7 @@ ulc_width_linebreaks (const char *s, size_t n, if (is_all_ascii (s, n)) { /* ASCII is a subset of UTF-8. */ - return u8_width_linebreaks ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, p); + return u8_width_linebreaks_internal ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, cr, p); } #endif /* We have a non-ASCII string and cannot convert it. @@ -127,9 +129,16 @@ ulc_width_linebreaks (const char *s, size_t n, const char *s_end = s + n; while (s < s_end) { - *p = ((o != NULL && *o == UC_BREAK_MANDATORY) || *s == '\n' + *p = ((o != NULL && *o == UC_BREAK_MANDATORY) + || *s == '\n' ? UC_BREAK_MANDATORY - : UC_BREAK_PROHIBITED); + : ((o != NULL && *o == UC_BREAK_CR_BEFORE_LF) + || (cr >= 0 + && *s == '\r' + && s + 1 < s_end + && *(s + 1) == '\n') + ? UC_BREAK_CR_BEFORE_LF + : UC_BREAK_PROHIBITED)); s++; p++; if (o != NULL) @@ -142,6 +151,30 @@ ulc_width_linebreaks (const char *s, size_t n, return start_column; } +#undef ulc_width_linebreaks + +int +ulc_width_linebreaks (const char *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return ulc_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, -1, p); +} + +int +ulc_width_linebreaks_v2 (const char *s, size_t n, + int width, int start_column, int at_end_columns, + const char *o, const char *encoding, + char *p) +{ + return ulc_width_linebreaks_internal (s, n, + width, start_column, at_end_columns, + o, encoding, LBP_CR, p); +} + #ifdef TEST @@ -209,7 +242,7 @@ main (int argc, char * argv[]) char *breaks = malloc (length); int i; - ulc_width_linebreaks (input, length, width, 0, 0, NULL, locale_charset (), breaks); + ulc_width_linebreaks_v2 (input, length, width, 0, 0, NULL, locale_charset (), breaks); for (i = 0; i < length; i++) { @@ -220,6 +253,8 @@ main (int argc, char * argv[]) break; case UC_BREAK_MANDATORY: break; + case UC_BREAK_CR_BEFORE_LF: + break; case UC_BREAK_PROHIBITED: break; default: diff --git a/modules/unilbrk/u16-possible-linebreaks b/modules/unilbrk/u16-possible-linebreaks index 1446d29b36..540285a1ba 100644 --- a/modules/unilbrk/u16-possible-linebreaks +++ b/modules/unilbrk/u16-possible-linebreaks @@ -2,6 +2,7 @@ Description: Line breaking of UTF-16 strings. Files: +lib/unilbrk/internal.h lib/unilbrk/u16-possible-linebreaks.c lib/uniwidth/cjk.h diff --git a/modules/unilbrk/u32-possible-linebreaks b/modules/unilbrk/u32-possible-linebreaks index b42848aa4f..0f123e90c3 100644 --- a/modules/unilbrk/u32-possible-linebreaks +++ b/modules/unilbrk/u32-possible-linebreaks @@ -2,6 +2,7 @@ Description: Line breaking of UTF-32 strings. Files: +lib/unilbrk/internal.h lib/unilbrk/u32-possible-linebreaks.c lib/uniwidth/cjk.h diff --git a/modules/unilbrk/u8-possible-linebreaks b/modules/unilbrk/u8-possible-linebreaks index 7e7f61e32f..bb1d2ca1f0 100644 --- a/modules/unilbrk/u8-possible-linebreaks +++ b/modules/unilbrk/u8-possible-linebreaks @@ -2,6 +2,7 @@ Description: Line breaking of UTF-8 strings. Files: +lib/unilbrk/internal.h lib/unilbrk/u8-possible-linebreaks.c lib/uniwidth/cjk.h diff --git a/modules/unilbrk/u8-width-linebreaks b/modules/unilbrk/u8-width-linebreaks index d2260f0614..daf1ca8fa1 100644 --- a/modules/unilbrk/u8-width-linebreaks +++ b/modules/unilbrk/u8-width-linebreaks @@ -2,6 +2,7 @@ Description: Line breaking of UTF-8 strings. Files: +lib/unilbrk/internal.h lib/unilbrk/u8-width-linebreaks.c Depends-on: diff --git a/tests/unilbrk/test-u16-possible-linebreaks.c b/tests/unilbrk/test-u16-possible-linebreaks.c index fedaa9d842..f582a0c752 100644 --- a/tests/unilbrk/test-u16-possible-linebreaks.c +++ b/tests/unilbrk/test-u16-possible-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (void (*my_u16_possible_linebreaks) (const uint16_t *, size_t, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - u16_possible_linebreaks (NULL, 0, "GB18030", NULL); + my_u16_possible_linebreaks (NULL, 0, "GB18030", NULL); { static const uint16_t input[61] = @@ -45,7 +46,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "GB18030", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "GB18030", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -64,7 +65,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "GB2312", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "GB2312", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -80,6 +81,23 @@ main () } } + /* CR LF handling. */ + { + static const uint16_t input[8] = + { 'a', '\n', 'b', '\r', 'c', '\r', '\n', 'd' }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 8; i++) + { + ASSERT (p[i] == (i == 1 || i == 3 || i == 6 ? UC_BREAK_MANDATORY : + i == 5 ? (version < 2 ? UC_BREAK_MANDATORY : UC_BREAK_CR_BEFORE_LF) : + UC_BREAK_PROHIBITED)); + } + free (p); + } + /* Test that a break is possible after a zero-width space followed by some regular spaces (rule LB8 in Unicode TR#14 revision 26). */ { @@ -87,7 +105,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 4; i++) { ASSERT (p[i] == (i == 3 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); @@ -105,7 +123,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 21; i++) { ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : @@ -125,7 +143,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 16; i++) { ASSERT (p[i] == (i == 14 ? UC_BREAK_MANDATORY : @@ -149,7 +167,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 39; i++) { ASSERT (p[i] == (i == 8 || i == 20 @@ -171,7 +189,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 8; i++) { ASSERT (p[i] == (i == 4 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); @@ -187,7 +205,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 10; i++) { ASSERT (p[i] == (i == 3 || i == 9 ? UC_BREAK_POSSIBLE : @@ -195,6 +213,14 @@ main () } free (p); } +} + +int +main () +{ + test_function (u16_possible_linebreaks, 2); +#undef u16_possible_linebreaks + test_function (u16_possible_linebreaks, 1); return 0; } diff --git a/tests/unilbrk/test-u16-width-linebreaks.c b/tests/unilbrk/test-u16-width-linebreaks.c index a4297b8252..73946f285d 100644 --- a/tests/unilbrk/test-u16-width-linebreaks.c +++ b/tests/unilbrk/test-u16-width-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (int (*my_u16_width_linebreaks) (const uint16_t *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - u16_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); + my_u16_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); { static const uint16_t input[61] = @@ -45,7 +46,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p); + my_u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -59,7 +60,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p); + my_u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -69,6 +70,14 @@ main () free (p); } } +} + +int +main () +{ + test_function (u16_width_linebreaks, 2); +#undef u16_width_linebreaks + test_function (u16_width_linebreaks, 1); return 0; } diff --git a/tests/unilbrk/test-u32-possible-linebreaks.c b/tests/unilbrk/test-u32-possible-linebreaks.c index 1d8dfb122b..fb435add2c 100644 --- a/tests/unilbrk/test-u32-possible-linebreaks.c +++ b/tests/unilbrk/test-u32-possible-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (void (*my_u32_possible_linebreaks) (const uint32_t *, size_t, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - u32_possible_linebreaks (NULL, 0, "GB18030", NULL); + my_u32_possible_linebreaks (NULL, 0, "GB18030", NULL); { static const uint32_t input[61] = @@ -45,7 +46,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "GB18030", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "GB18030", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -64,7 +65,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "GB2312", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "GB2312", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -80,6 +81,23 @@ main () } } + /* CR LF handling. */ + { + static const uint32_t input[8] = + { 'a', '\n', 'b', '\r', 'c', '\r', '\n', 'd' }; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 8; i++) + { + ASSERT (p[i] == (i == 1 || i == 3 || i == 6 ? UC_BREAK_MANDATORY : + i == 5 ? (version < 2 ? UC_BREAK_MANDATORY : UC_BREAK_CR_BEFORE_LF) : + UC_BREAK_PROHIBITED)); + } + free (p); + } + /* Test that a break is possible after a zero-width space followed by some regular spaces (rule LB8 in Unicode TR#14 revision 26). */ { @@ -87,7 +105,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 4; i++) { ASSERT (p[i] == (i == 3 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); @@ -105,7 +123,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 21; i++) { ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : @@ -125,7 +143,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 16; i++) { ASSERT (p[i] == (i == 14 ? UC_BREAK_MANDATORY : @@ -149,7 +167,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 37; i++) { ASSERT (p[i] == (i == 8 || i == 20 @@ -171,7 +189,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 4; i++) { ASSERT (p[i] == (i == 2 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); @@ -187,7 +205,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 10; i++) { ASSERT (p[i] == (i == 3 || i == 9 ? UC_BREAK_POSSIBLE : @@ -195,6 +213,14 @@ main () } free (p); } +} + +int +main () +{ + test_function (u32_possible_linebreaks, 2); +#undef u32_possible_linebreaks + test_function (u32_possible_linebreaks, 1); return 0; } diff --git a/tests/unilbrk/test-u32-width-linebreaks.c b/tests/unilbrk/test-u32-width-linebreaks.c index bc38582f9f..7e7dd25cf1 100644 --- a/tests/unilbrk/test-u32-width-linebreaks.c +++ b/tests/unilbrk/test-u32-width-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (int (*my_u32_width_linebreaks) (const uint32_t *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - u32_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); + my_u32_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); { static const uint32_t input[61] = @@ -45,7 +46,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p); + my_u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -59,7 +60,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p); + my_u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p); for (i = 0; i < 61; i++) { ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY : @@ -69,6 +70,14 @@ main () free (p); } } +} + +int +main () +{ + test_function (u32_width_linebreaks, 2); +#undef u32_width_linebreaks + test_function (u32_width_linebreaks, 1); return 0; } diff --git a/tests/unilbrk/test-u8-possible-linebreaks.c b/tests/unilbrk/test-u8-possible-linebreaks.c index cad7b7afa3..f0ddc02500 100644 --- a/tests/unilbrk/test-u8-possible-linebreaks.c +++ b/tests/unilbrk/test-u8-possible-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (void (*my_u8_possible_linebreaks) (const uint8_t *, size_t, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - u8_possible_linebreaks (NULL, 0, "GB18030", NULL); + my_u8_possible_linebreaks (NULL, 0, "GB18030", NULL); { static const uint8_t input[91] = @@ -39,7 +40,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "GB18030", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "GB18030", p); for (i = 0; i < 91; i++) { ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY : @@ -58,7 +59,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "GB2312", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "GB2312", p); for (i = 0; i < 91; i++) { ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY : @@ -74,6 +75,22 @@ main () } } + /* CR LF handling. */ + { + static const uint8_t input[8] = "a\nb\rc\r\nd"; + char *p = (char *) malloc (SIZEOF (input)); + size_t i; + + my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + for (i = 0; i < 8; i++) + { + ASSERT (p[i] == (i == 1 || i == 3 || i == 6 ? UC_BREAK_MANDATORY : + i == 5 ? (version < 2 ? UC_BREAK_MANDATORY : UC_BREAK_CR_BEFORE_LF) : + UC_BREAK_PROHIBITED)); + } + free (p); + } + /* Test that a break is possible after a zero-width space followed by some regular spaces (rule LB8 in Unicode TR#14 revision 26). */ { @@ -81,7 +98,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 6; i++) { ASSERT (p[i] == (i == 5 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); @@ -95,7 +112,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 21; i++) { ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : @@ -112,7 +129,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 24; i++) { ASSERT (p[i] == (i == 21 ? UC_BREAK_MANDATORY : @@ -134,7 +151,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 101; i++) { ASSERT (p[i] == (i == 24 || i == 58 @@ -156,7 +173,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 16; i++) { ASSERT (p[i] == (i == 8 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED)); @@ -172,7 +189,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 15; i++) { ASSERT (p[i] == (i == 3 || i == 13 ? UC_BREAK_POSSIBLE : @@ -180,6 +197,14 @@ main () } free (p); } +} + +int +main () +{ + test_function (u8_possible_linebreaks, 2); +#undef u8_possible_linebreaks + test_function (u8_possible_linebreaks, 1); return 0; } diff --git a/tests/unilbrk/test-u8-width-linebreaks.c b/tests/unilbrk/test-u8-width-linebreaks.c index 4deb5451bc..250ba2be37 100644 --- a/tests/unilbrk/test-u8-width-linebreaks.c +++ b/tests/unilbrk/test-u8-width-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (int (*my_u8_width_linebreaks) (const uint8_t *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - u8_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); + my_u8_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); { static const uint8_t input[91] = @@ -39,7 +40,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p); + my_u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p); for (i = 0; i < 91; i++) { ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY : @@ -53,7 +54,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p); + my_u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p); for (i = 0; i < 91; i++) { ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY : @@ -63,6 +64,14 @@ main () free (p); } } +} + +int +main () +{ + test_function (u8_width_linebreaks, 2); +#undef u8_width_linebreaks + test_function (u8_width_linebreaks, 1); return 0; } diff --git a/tests/unilbrk/test-ulc-possible-linebreaks.c b/tests/unilbrk/test-ulc-possible-linebreaks.c index f3ad7f9222..95742b9df1 100644 --- a/tests/unilbrk/test-ulc-possible-linebreaks.c +++ b/tests/unilbrk/test-ulc-possible-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (void (*my_ulc_possible_linebreaks) (const char *, size_t, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - ulc_possible_linebreaks (NULL, 0, "GB18030", NULL); + my_ulc_possible_linebreaks (NULL, 0, "GB18030", NULL); #if HAVE_ICONV { @@ -38,7 +39,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - ulc_possible_linebreaks (input, SIZEOF (input), "ISO-8859-1", p); + my_ulc_possible_linebreaks (input, SIZEOF (input), "ISO-8859-1", p); for (i = 0; i < 36; i++) { ASSERT (p[i] == (i == 35 ? UC_BREAK_MANDATORY : @@ -55,7 +56,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - ulc_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); + my_ulc_possible_linebreaks (input, SIZEOF (input), "UTF-8", p); for (i = 0; i < 21; i++) { ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE : @@ -64,6 +65,14 @@ main () free (p); } #endif +} + +int +main () +{ + test_function (ulc_possible_linebreaks, 2); +#undef ulc_possible_linebreaks + test_function (ulc_possible_linebreaks, 1); return 0; } diff --git a/tests/unilbrk/test-ulc-width-linebreaks.c b/tests/unilbrk/test-ulc-width-linebreaks.c index 78fbe7e5dc..7d742eec2a 100644 --- a/tests/unilbrk/test-ulc-width-linebreaks.c +++ b/tests/unilbrk/test-ulc-width-linebreaks.c @@ -24,11 +24,12 @@ #include "macros.h" -int -main () +static void +test_function (int (*my_ulc_width_linebreaks) (const char *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT), + int version) { /* Test case n = 0. */ - ulc_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); + my_ulc_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL); #if HAVE_ICONV { @@ -38,7 +39,7 @@ main () char *p = (char *) malloc (SIZEOF (input)); size_t i; - ulc_width_linebreaks (input, SIZEOF (input), 12, 0, 0, NULL, "ISO-8859-1", p); + my_ulc_width_linebreaks (input, SIZEOF (input), 12, 0, 0, NULL, "ISO-8859-1", p); for (i = 0; i < 36; i++) { ASSERT (p[i] == (i == 35 ? UC_BREAK_MANDATORY : @@ -48,6 +49,14 @@ main () free (p); } #endif +} + +int +main () +{ + test_function (ulc_width_linebreaks, 2); +#undef ulc_width_linebreaks + test_function (ulc_width_linebreaks, 1); return 0; }