* lib/gen-uni-tables.c (LBP_CR, LBP_LF): New enum values.
(get_lbp): For '\r' and '\n', return LBP_CR and LBP_LF, respectively,
instead of LBP_BK.
(debug_output_lbp, fill_org_lbp, debug_output_org_lbp,
lbp_value_to_string): Handle LBP_CR, LBP_LF.
* lib/unilbrk/lbrkprop2.h: Regenerated.
* lib/unilbrk/lbrktables.h (LBP_CR, LBP_LF): New enum values.
* lib/unilbrk.in.h (UC_BREAK_CR_BEFORE_LF): New enum value.
(u8_possible_linebreaks, u16_possible_linebreaks,
u32_possible_linebreaks, ulc_possible_linebreaks, u8_width_linebreaks,
u16_width_linebreaks, u32_width_linebreaks, ulc_width_linebreaks):
Declare also a _v2 variant, and use a C macro to activate it by default.
* lib/unilbrk/internal.h: New file.
* lib/unilbrk/u8-possible-linebreaks.c: Include unilbrk/internal.h.
(u8_possible_linebreaks_loop): Renamed from u8_possible_linebreaks. Add
'cr' parameter.
(u8_possible_linebreaks, u8_possible_linebreaks_v2): New functions.
(main): Update.
* lib/unilbrk/u16-possible-linebreaks.c: Include unilbrk/internal.h.
(u16_possible_linebreaks_loop): Renamed from u16_possible_linebreaks.
Add 'cr' parameter.
(u16_possible_linebreaks, u16_possible_linebreaks_v2): New functions.
* lib/unilbrk/u32-possible-linebreaks.c: Include unilbrk/internal.h.
(u32_possible_linebreaks_loop): Renamed from u32_possible_linebreaks.
Add 'cr' parameter.
(u32_possible_linebreaks, u32_possible_linebreaks_v2): New functions.
* lib/unilbrk/ulc-possible-linebreaks.c: Include unilbrk/internal.h,
unilbrk/lbrktables.h.
(ulc_possible_linebreaks_internal): Renamed from
ulc_possible_linebreaks. Add 'cr' parameter.
(ulc_possible_linebreaks, ulc_possible_linebreaks_v2): New functions.
(main): Update.
* lib/unilbrk/u8-width-linebreaks.c: Include unilbrk/internal.h,
unilbrk/lbrktables.h.
(u8_width_linebreaks_internal): Renamed from u8_width_linebreaks. Add
'cr' parameter.
(u8_width_linebreaks, u8_width_linebreaks_v2): New functions.
(main): Update.
* lib/unilbrk/u16-width-linebreaks.c: Include unilbrk/internal.h,
unilbrk/lbrktables.h.
(u16_width_linebreaks_internal): Renamed from u16_width_linebreaks. Add
'cr' parameter.
(u16_width_linebreaks, u16_width_linebreaks_v2): New functions.
* lib/unilbrk/u32-width-linebreaks.c: Include unilbrk/internal.h,
unilbrk/lbrktables.h.
(u32_width_linebreaks_internal): Renamed from u32_width_linebreaks. Add
'cr' parameter.
(u32_width_linebreaks, u32_width_linebreaks_v2): New functions.
* lib/unilbrk/ulc-width-linebreaks.c: Include unilbrk/internal.h,
unilbrk/lbrktables.h.
(ulc_width_linebreaks_internal): Renamed from ulc_width_linebreaks. Add
'cr' parameter.
(ulc_width_linebreaks, ulc_width_linebreaks_v2): New functions.
(main): Update.
* modules/unilbrk/u8-possible-linebreaks (Files): Add
lib/unilbrk/internal.h.
* modules/unilbrk/u16-possible-linebreaks (Files): Likewise.
* modules/unilbrk/u32-possible-linebreaks (Files): Likewise.
* modules/unilbrk/u8-width-linebreaks (Files): Likewise.
* tests/unilbrk/test-u8-possible-linebreaks.c (test_function): New
function, extracted from main. Add a test of CR-LF handling.
(main): Invoke it twice.
* tests/unilbrk/test-u16-possible-linebreaks.c (test_function): New
function, extracted from main. Add a test of CR-LF handling.
(main): Invoke it twice.
* tests/unilbrk/test-u32-possible-linebreaks.c (test_function): New
function, extracted from main. Add a test of CR-LF handling.
(main): Invoke it twice.
* tests/unilbrk/test-ulc-possible-linebreaks.c (test_function): New
function, extracted from main.
(main): Invoke it twice.
* tests/unilbrk/test-u8-width-linebreaks.c (test_function): New
function, extracted from main.
(main): Invoke it twice.
* tests/unilbrk/test-u16-width-linebreaks.c (test_function): New
function, extracted from main.
(main): Invoke it twice.
* tests/unilbrk/test-u32-width-linebreaks.c (test_function): New
function, extracted from main.
(main): Invoke it twice.
* tests/unilbrk/test-ulc-width-linebreaks.c (test_function): New
function, extracted from main.
(main): Invoke it twice.
+2021-12-29 Bruno Haible <bruno@clisp.org>
+
+ unilbrk: Improve support for strings that contain CR-LF sequences.
+ * lib/gen-uni-tables.c (LBP_CR, LBP_LF): New enum values.
+ (get_lbp): For '\r' and '\n', return LBP_CR and LBP_LF, respectively,
+ instead of LBP_BK.
+ (debug_output_lbp, fill_org_lbp, debug_output_org_lbp,
+ lbp_value_to_string): Handle LBP_CR, LBP_LF.
+ * lib/unilbrk/lbrkprop2.h: Regenerated.
+ * lib/unilbrk/lbrktables.h (LBP_CR, LBP_LF): New enum values.
+ * lib/unilbrk.in.h (UC_BREAK_CR_BEFORE_LF): New enum value.
+ (u8_possible_linebreaks, u16_possible_linebreaks,
+ u32_possible_linebreaks, ulc_possible_linebreaks, u8_width_linebreaks,
+ u16_width_linebreaks, u32_width_linebreaks, ulc_width_linebreaks):
+ Declare also a _v2 variant, and use a C macro to activate it by default.
+ * lib/unilbrk/internal.h: New file.
+ * lib/unilbrk/u8-possible-linebreaks.c: Include unilbrk/internal.h.
+ (u8_possible_linebreaks_loop): Renamed from u8_possible_linebreaks. Add
+ 'cr' parameter.
+ (u8_possible_linebreaks, u8_possible_linebreaks_v2): New functions.
+ (main): Update.
+ * lib/unilbrk/u16-possible-linebreaks.c: Include unilbrk/internal.h.
+ (u16_possible_linebreaks_loop): Renamed from u16_possible_linebreaks.
+ Add 'cr' parameter.
+ (u16_possible_linebreaks, u16_possible_linebreaks_v2): New functions.
+ * lib/unilbrk/u32-possible-linebreaks.c: Include unilbrk/internal.h.
+ (u32_possible_linebreaks_loop): Renamed from u32_possible_linebreaks.
+ Add 'cr' parameter.
+ (u32_possible_linebreaks, u32_possible_linebreaks_v2): New functions.
+ * lib/unilbrk/ulc-possible-linebreaks.c: Include unilbrk/internal.h,
+ unilbrk/lbrktables.h.
+ (ulc_possible_linebreaks_internal): Renamed from
+ ulc_possible_linebreaks. Add 'cr' parameter.
+ (ulc_possible_linebreaks, ulc_possible_linebreaks_v2): New functions.
+ (main): Update.
+ * lib/unilbrk/u8-width-linebreaks.c: Include unilbrk/internal.h,
+ unilbrk/lbrktables.h.
+ (u8_width_linebreaks_internal): Renamed from u8_width_linebreaks. Add
+ 'cr' parameter.
+ (u8_width_linebreaks, u8_width_linebreaks_v2): New functions.
+ (main): Update.
+ * lib/unilbrk/u16-width-linebreaks.c: Include unilbrk/internal.h,
+ unilbrk/lbrktables.h.
+ (u16_width_linebreaks_internal): Renamed from u16_width_linebreaks. Add
+ 'cr' parameter.
+ (u16_width_linebreaks, u16_width_linebreaks_v2): New functions.
+ * lib/unilbrk/u32-width-linebreaks.c: Include unilbrk/internal.h,
+ unilbrk/lbrktables.h.
+ (u32_width_linebreaks_internal): Renamed from u32_width_linebreaks. Add
+ 'cr' parameter.
+ (u32_width_linebreaks, u32_width_linebreaks_v2): New functions.
+ * lib/unilbrk/ulc-width-linebreaks.c: Include unilbrk/internal.h,
+ unilbrk/lbrktables.h.
+ (ulc_width_linebreaks_internal): Renamed from ulc_width_linebreaks. Add
+ 'cr' parameter.
+ (ulc_width_linebreaks, ulc_width_linebreaks_v2): New functions.
+ (main): Update.
+ * modules/unilbrk/u8-possible-linebreaks (Files): Add
+ lib/unilbrk/internal.h.
+ * modules/unilbrk/u16-possible-linebreaks (Files): Likewise.
+ * modules/unilbrk/u32-possible-linebreaks (Files): Likewise.
+ * modules/unilbrk/u8-width-linebreaks (Files): Likewise.
+ * tests/unilbrk/test-u8-possible-linebreaks.c (test_function): New
+ function, extracted from main. Add a test of CR-LF handling.
+ (main): Invoke it twice.
+ * tests/unilbrk/test-u16-possible-linebreaks.c (test_function): New
+ function, extracted from main. Add a test of CR-LF handling.
+ (main): Invoke it twice.
+ * tests/unilbrk/test-u32-possible-linebreaks.c (test_function): New
+ function, extracted from main. Add a test of CR-LF handling.
+ (main): Invoke it twice.
+ * tests/unilbrk/test-ulc-possible-linebreaks.c (test_function): New
+ function, extracted from main.
+ (main): Invoke it twice.
+ * tests/unilbrk/test-u8-width-linebreaks.c (test_function): New
+ function, extracted from main.
+ (main): Invoke it twice.
+ * tests/unilbrk/test-u16-width-linebreaks.c (test_function): New
+ function, extracted from main.
+ (main): Invoke it twice.
+ * tests/unilbrk/test-u32-width-linebreaks.c (test_function): New
+ function, extracted from main.
+ (main): Invoke it twice.
+ * tests/unilbrk/test-ulc-width-linebreaks.c (test_function): New
+ function, extracted from main.
+ (main): Invoke it twice.
+
2021-12-29 Bruno Haible <bruno@clisp.org>
uniwbrk: Add unit tests of CR-LF sequences.
{
/* Values >= 30 are resolved at run time. */
LBP_BK = 30, /* mandatory break */
-/*LBP_CR, carriage return - not used here because it's a DOSism */
-/*LBP_LF, line feed - not used here because it's a DOSism */
- LBP_CM = 31, /* attached characters and combining marks */
+ LBP_CR = 31, /* carriage return */
+ LBP_LF = 32, /* line feed */
+ LBP_CM = 33, /* attached characters and combining marks */
/*LBP_NL, next line - not used here because it's equivalent to LBP_BK */
/*LBP_SG, surrogates - not used here because they are not characters */
LBP_WJ = 0, /* word joiner */
- LBP_ZW = 32, /* zero width space */
+ LBP_ZW = 34, /* zero width space */
LBP_GL = 1, /* non-breaking (glue) */
- LBP_SP = 33, /* space */
+ LBP_SP = 35, /* space */
LBP_B2 = 2, /* break opportunity before and after */
LBP_BA = 3, /* break opportunity after */
LBP_BB = 4, /* break opportunity before */
LBP_HY = 5, /* hyphen */
- LBP_CB = 34, /* contingent break opportunity */
+ LBP_CB = 36, /* contingent break opportunity */
LBP_CL = 6, /* closing punctuation */
LBP_CP = 7, /* closing parenthesis */
LBP_EX = 8, /* exclamation/interrogation */
LBP_PO = 15, /* postfix (numeric) */
LBP_PR = 16, /* prefix (numeric) */
LBP_SY = 17, /* symbols allowing breaks */
- LBP_AI = 35, /* ambiguous (alphabetic or ideograph) */
+ LBP_AI = 37, /* ambiguous (alphabetic or ideograph) */
LBP_AL = 18, /* ordinary alphabetic and symbol characters */
/*LBP_CJ, conditional Japanese starter, resolved to NS */
LBP_H2 = 19, /* Hangul LV syllable */
LBP_JV = 23, /* Hangul V Jamo */
LBP_JT = 24, /* Hangul T Jamo */
LBP_RI = 26, /* regional indicator */
- LBP_SA = 36, /* complex context (South East Asian) */
+ LBP_SA = 38, /* complex context (South East Asian) */
LBP_ZWJ = 27, /* zero width joiner */
LBP_EB = 28, /* emoji base */
LBP_EM = 29, /* emoji modifier */
- LBP_XX = 37 /* unknown */
+ LBP_XX = 39 /* unknown */
};
/* Returns the line breaking classification for ch, as a bit mask. */
if (unicode_attributes[ch].name != NULL)
{
/* mandatory break */
- if (ch == 0x000A || ch == 0x000D || ch == 0x0085 /* newline */
+ if (ch == 0x000A)
+ attr |= (int64_t) 1 << LBP_LF;
+ if (ch == 0x000D)
+ attr |= (int64_t) 1 << LBP_CR;
+ if (ch == 0x0085 /* newline */
|| ch == 0x000C /* form feed */
|| ch == 0x000B /* line tabulation */
|| ch == 0x2028 /* LINE SEPARATOR */
&& ch != 0x110BD /* KAITHI NUMBER SIGN */
&& ch != 0x08E2 /* ARABIC DISPUTED END OF AYAH */)
|| ch == 0x3035 /* VERTICAL KANA REPEAT MARK LOWER HALF */)
- if (!(attr & (((int64_t) 1 << LBP_BK) | ((int64_t) 1 << LBP_BA) | ((int64_t) 1 << LBP_GL) | ((int64_t) 1 << LBP_SA) | ((int64_t) 1 << LBP_WJ) | ((int64_t) 1 << LBP_ZW) | ((int64_t) 1 << LBP_ZWJ))))
+ if (!(attr & (((int64_t) 1 << LBP_BK) | ((int64_t) 1 << LBP_CR) | ((int64_t) 1 << LBP_LF) | ((int64_t) 1 << LBP_BA) | ((int64_t) 1 << LBP_GL) | ((int64_t) 1 << LBP_SA) | ((int64_t) 1 << LBP_WJ) | ((int64_t) 1 << LBP_ZW) | ((int64_t) 1 << LBP_ZWJ))))
attr |= (int64_t) 1 << LBP_CM;
/* ideographic */
#define PRINT_BIT(attr,bit) \
if (attr & ((int64_t) 1 << bit)) fprintf (stream, " " #bit);
PRINT_BIT(attr,LBP_BK);
+ PRINT_BIT(attr,LBP_CR);
+ PRINT_BIT(attr,LBP_LF);
PRINT_BIT(attr,LBP_CM);
PRINT_BIT(attr,LBP_WJ);
PRINT_BIT(attr,LBP_ZW);
#define TRY(bit) else if (strcmp (field1, #bit + 4) == 0) value = bit;
if (false) {}
TRY(LBP_BK)
+ TRY(LBP_CR)
+ TRY(LBP_LF)
TRY(LBP_CM)
TRY(LBP_WJ)
TRY(LBP_ZW)
TRY(LBP_EM)
TRY(LBP_XX)
#undef TRY
- else if (strcmp (field1, "LF") == 0) value = LBP_BK;
- else if (strcmp (field1, "CR") == 0) value = LBP_BK;
else if (strcmp (field1, "NL") == 0) value = LBP_BK;
else if (strcmp (field1, "SG") == 0) value = LBP_XX;
else if (strcmp (field1, "CJ") == 0) value = LBP_NS;
#define PRINT_BIT(attr,bit) \
if (attr == bit) fprintf (stream, " " #bit);
PRINT_BIT(attr,LBP_BK);
+ PRINT_BIT(attr,LBP_CR);
+ PRINT_BIT(attr,LBP_LF);
PRINT_BIT(attr,LBP_CM);
PRINT_BIT(attr,LBP_WJ);
PRINT_BIT(attr,LBP_ZW);
{
#define CASE(x) case x: value_string = #x; break;
CASE(LBP_BK);
+ CASE(LBP_CR);
+ CASE(LBP_LF);
CASE(LBP_CM);
CASE(LBP_WJ);
CASE(LBP_ZW);
UC_BREAK_PROHIBITED,
UC_BREAK_POSSIBLE,
UC_BREAK_MANDATORY,
- UC_BREAK_HYPHENATION
+ UC_BREAK_HYPHENATION,
+ UC_BREAK_CR_BEFORE_LF /* only used in _v2 or later */
};
/* Determine the line break points in S, and store the result at p[0..n-1].
p[i] = UC_BREAK_MANDATORY means that s[i] is a line break character.
+ p[i] = UC_BREAK_CR_BEFORE_LF means that s[i] and s[i+1] is the CR-LF
+ character sequence. (Only used in _v2 or later.)
p[i] = UC_BREAK_POSSIBLE means that a line break may be inserted between
s[i-1] and s[i].
p[i] = UC_BREAK_HYPHENATION means that a hyphen and a line break may be
extern void
u8_possible_linebreaks (const uint8_t *s, size_t n,
const char *encoding, char *_UC_RESTRICT p);
+extern void
+ u8_possible_linebreaks_v2 (const uint8_t *s, size_t n,
+ const char *encoding, char *_UC_RESTRICT p);
+#define u8_possible_linebreaks u8_possible_linebreaks_v2
+
extern void
u16_possible_linebreaks (const uint16_t *s, size_t n,
const char *encoding, char *_UC_RESTRICT p);
+extern void
+ u16_possible_linebreaks_v2 (const uint16_t *s, size_t n,
+ const char *encoding, char *_UC_RESTRICT p);
+#define u16_possible_linebreaks u16_possible_linebreaks_v2
+
extern void
u32_possible_linebreaks (const uint32_t *s, size_t n,
const char *encoding, char *_UC_RESTRICT p);
+extern void
+ u32_possible_linebreaks_v2 (const uint32_t *s, size_t n,
+ const char *encoding, char *_UC_RESTRICT p);
+#define u32_possible_linebreaks u32_possible_linebreaks_v2
+
extern void
ulc_possible_linebreaks (const char *s, size_t n,
const char *encoding, char *_UC_RESTRICT p);
+extern void
+ ulc_possible_linebreaks_v2 (const char *s, size_t n,
+ const char *encoding, char *_UC_RESTRICT p);
+#define ulc_possible_linebreaks ulc_possible_linebreaks_v2
/* Choose the best line breaks, assuming the uc_width function.
The string is s[0..n-1]. The maximum number of columns per line is given
int start_column, int at_end_columns,
const char *o, const char *encoding,
char *_UC_RESTRICT p);
+extern int
+ u8_width_linebreaks_v2 (const uint8_t *s, size_t n, int width,
+ int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *_UC_RESTRICT p);
+#define u8_width_linebreaks u8_width_linebreaks_v2
+
extern int
u16_width_linebreaks (const uint16_t *s, size_t n, int width,
int start_column, int at_end_columns,
const char *o, const char *encoding,
char *_UC_RESTRICT p);
+extern int
+ u16_width_linebreaks_v2 (const uint16_t *s, size_t n, int width,
+ int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *_UC_RESTRICT p);
+#define u16_width_linebreaks u16_width_linebreaks_v2
+
extern int
u32_width_linebreaks (const uint32_t *s, size_t n, int width,
int start_column, int at_end_columns,
const char *o, const char *encoding,
char *_UC_RESTRICT p);
+extern int
+ u32_width_linebreaks_v2 (const uint32_t *s, size_t n, int width,
+ int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *_UC_RESTRICT p);
+#define u32_width_linebreaks u32_width_linebreaks_v2
+
extern int
ulc_width_linebreaks (const char *s, size_t n, int width,
int start_column, int at_end_columns,
const char *o, const char *encoding,
char *_UC_RESTRICT p);
+extern int
+ ulc_width_linebreaks_v2 (const char *s, size_t n, int width,
+ int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *_UC_RESTRICT p);
+#define ulc_width_linebreaks ulc_width_linebreaks_v2
#ifdef __cplusplus
--- /dev/null
+/* Internal functions for line breaking of Unicode strings.
+ Copyright (C) 2001-2003, 2005-2021 Free Software Foundation, Inc.
+ Written by Bruno Haible <bruno@clisp.org>, 2021.
+
+ This file is free software.
+ It is dual-licensed under "the GNU LGPLv3+ or the GNU GPLv2+".
+ You can redistribute it and/or modify it under either
+ - the terms of the GNU Lesser General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version, or
+ - the terms of the GNU General Public License as published by the
+ Free Software Foundation; either version 2, or (at your option)
+ any later version, or
+ - the same dual license "the GNU LGPLv3+ or the GNU GPLv2+".
+
+ This file is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License and the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License and of the GNU General Public License along with this
+ program. If not, see <https://www.gnu.org/licenses/>. */
+
+extern void
+ u8_possible_linebreaks_loop (const uint8_t *s, size_t n,
+ const char *encoding, int cr,
+ char *_UC_RESTRICT p);
+extern void
+ u16_possible_linebreaks_loop (const uint16_t *s, size_t n,
+ const char *encoding, int cr,
+ char *_UC_RESTRICT p);
+extern void
+ u32_possible_linebreaks_loop (const uint32_t *s, size_t n,
+ const char *encoding, int cr,
+ char *_UC_RESTRICT p);
+
+extern int
+ u8_width_linebreaks_internal (const uint8_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding, int cr,
+ char *p);
},
{
LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM,
- LBP_CM, LBP_BA, LBP_BK, LBP_BK, LBP_BK, LBP_BK, LBP_CM, LBP_CM,
+ LBP_CM, LBP_BA, LBP_LF, LBP_BK, LBP_BK, LBP_CR, LBP_CM, LBP_CM,
LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM,
LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM, LBP_CM,
LBP_SP, LBP_EX, LBP_QU, LBP_AL, LBP_PR, LBP_PO, LBP_AL, LBP_QU,
{
/* Values >= 30 are resolved at run time. */
LBP_BK = 30, /* mandatory break */
-/*LBP_CR, carriage return - not used here because it's a DOSism */
-/*LBP_LF, line feed - not used here because it's a DOSism */
- LBP_CM = 31, /* attached characters and combining marks */
+ LBP_CR = 31, /* carriage return */
+ LBP_LF = 32, /* line feed */
+ LBP_CM = 33, /* attached characters and combining marks */
/*LBP_NL, next line - not used here because it's equivalent to LBP_BK */
/*LBP_SG, surrogates - not used here because they are not characters */
LBP_WJ = 0, /* word joiner */
- LBP_ZW = 32, /* zero width space */
+ LBP_ZW = 34, /* zero width space */
LBP_GL = 1, /* non-breaking (glue) */
- LBP_SP = 33, /* space */
+ LBP_SP = 35, /* space */
LBP_B2 = 2, /* break opportunity before and after */
LBP_BA = 3, /* break opportunity after */
LBP_BB = 4, /* break opportunity before */
LBP_HY = 5, /* hyphen */
- LBP_CB = 34, /* contingent break opportunity */
+ LBP_CB = 36, /* contingent break opportunity */
LBP_CL = 6, /* closing punctuation */
LBP_CP = 7, /* closing parenthesis */
LBP_EX = 8, /* exclamation/interrogation */
LBP_PO = 15, /* postfix (numeric) */
LBP_PR = 16, /* prefix (numeric) */
LBP_SY = 17, /* symbols allowing breaks */
- LBP_AI = 35, /* ambiguous (alphabetic or ideograph) */
+ LBP_AI = 37, /* ambiguous (alphabetic or ideograph) */
LBP_AL = 18, /* ordinary alphabetic and symbol characters */
/*LBP_CJ, conditional Japanese starters, resolved to NS */
LBP_H2 = 19, /* Hangul LV syllable */
LBP_JV = 23, /* Hangul V Jamo */
LBP_JT = 24, /* Hangul T Jamo */
LBP_RI = 26, /* regional indicator */
- LBP_SA = 36, /* complex context (South East Asian) */
+ LBP_SA = 38, /* complex context (South East Asian) */
LBP_ZWJ = 27, /* zero width joiner */
LBP_EB = 28, /* emoji base */
LBP_EM = 29, /* emoji modifier */
- LBP_XX = 37, /* unknown */
+ LBP_XX = 39, /* unknown */
/* Artificial values that exist only at runtime, not in the tables. */
LBP_HL_BA = 100
};
/* Specification. */
#include "unilbrk.h"
+#include "unilbrk/internal.h"
#include <stdlib.h>
#include <string.h>
#include "unistr.h"
void
-u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding, char *p)
+u16_possible_linebreaks_loop (const uint16_t *s, size_t n, const char *encoding,
+ int cr, char *p)
{
if (n > 0)
{
int count = u16_mbtouc_unsafe (&uc, s, s_end - s);
int prop = unilbrkprop_lookup (uc);
- if (prop == LBP_BK)
+ if (prop == LBP_BK || prop == LBP_LF || prop == LBP_CR)
{
/* (LB4,LB5,LB6) Mandatory break. */
*p = UC_BREAK_MANDATORY;
- prev_prop = LBP_BK;
+ /* cr is either LBP_CR or -1. In the first case, recognize
+ a CR-LF sequence. */
+ if (prev_prop == cr && prop == LBP_LF)
+ p[-1] = UC_BREAK_CR_BEFORE_LF;
+ prev_prop = prop;
last_prop = LBP_BK;
seen_space = NULL;
}
while (s < s_end);
}
}
+
+#undef u16_possible_linebreaks
+
+void
+u16_possible_linebreaks (const uint16_t *s, size_t n, const char *encoding,
+ char *p)
+{
+ u16_possible_linebreaks_loop (s, n, encoding, -1, p);
+}
+
+void
+u16_possible_linebreaks_v2 (const uint16_t *s, size_t n, const char *encoding,
+ char *p)
+{
+ u16_possible_linebreaks_loop (s, n, encoding, LBP_CR, p);
+}
/* Specification. */
#include "unilbrk.h"
+#include "unilbrk/internal.h"
+#include "unilbrk/lbrktables.h"
#include "unistr.h"
#include "uniwidth.h"
-int
-u16_width_linebreaks (const uint16_t *s, size_t n,
- int width, int start_column, int at_end_columns,
- const char *o, const char *encoding,
- char *p)
+static int
+u16_width_linebreaks_internal (const uint16_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding, int cr,
+ char *p)
{
const uint16_t *s_end;
char *last_p;
int last_column;
int piece_width;
- u16_possible_linebreaks (s, n, encoding, p);
+ u16_possible_linebreaks_loop (s, n, encoding, cr, p);
s_end = s + n;
last_p = NULL;
if (o != NULL && *o != UC_BREAK_UNDEFINED)
*p = *o;
- if (*p == UC_BREAK_POSSIBLE || *p == UC_BREAK_MANDATORY)
+ if (*p == UC_BREAK_POSSIBLE
+ || *p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF)
{
/* An atomic piece of text ends here. */
if (last_p != NULL && last_column + piece_width > width)
}
}
- if (*p == UC_BREAK_MANDATORY)
+ if (*p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF)
{
/* uc is a line break character. */
/* Start a new piece at column 0. */
return last_column + piece_width;
}
+
+#undef u16_width_linebreaks
+
+int
+u16_width_linebreaks (const uint16_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return u16_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, -1, p);
+}
+
+int
+u16_width_linebreaks_v2 (const uint16_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return u16_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, LBP_CR, p);
+}
/* Specification. */
#include "unilbrk.h"
+#include "unilbrk/internal.h"
#include <stdlib.h>
#include "uniwidth/cjk.h"
void
-u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding, char *p)
+u32_possible_linebreaks_loop (const uint32_t *s, size_t n, const char *encoding,
+ int cr, char *p)
{
if (n > 0)
{
ucs4_t uc = *s;
int prop = unilbrkprop_lookup (uc);
- if (prop == LBP_BK)
+ if (prop == LBP_BK || prop == LBP_LF || prop == LBP_CR)
{
/* (LB4,LB5,LB6) Mandatory break. */
*p = UC_BREAK_MANDATORY;
- prev_prop = LBP_BK;
+ /* cr is either LBP_CR or -1. In the first case, recognize
+ a CR-LF sequence. */
+ if (prev_prop == cr && prop == LBP_LF)
+ p[-1] = UC_BREAK_CR_BEFORE_LF;
+ prev_prop = prop;
last_prop = LBP_BK;
seen_space = NULL;
}
while (s < s_end);
}
}
+
+#undef u32_possible_linebreaks
+
+void
+u32_possible_linebreaks (const uint32_t *s, size_t n, const char *encoding,
+ char *p)
+{
+ u32_possible_linebreaks_loop (s, n, encoding, -1, p);
+}
+
+void
+u32_possible_linebreaks_v2 (const uint32_t *s, size_t n, const char *encoding,
+ char *p)
+{
+ u32_possible_linebreaks_loop (s, n, encoding, LBP_CR, p);
+}
/* Specification. */
#include "unilbrk.h"
+#include "unilbrk/internal.h"
+#include "unilbrk/lbrktables.h"
#include "uniwidth.h"
-int
-u32_width_linebreaks (const uint32_t *s, size_t n,
- int width, int start_column, int at_end_columns,
- const char *o, const char *encoding,
- char *p)
+static int
+u32_width_linebreaks_internal (const uint32_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding, int cr,
+ char *p)
{
const uint32_t *s_end;
char *last_p;
int last_column;
int piece_width;
- u32_possible_linebreaks (s, n, encoding, p);
+ u32_possible_linebreaks_loop (s, n, encoding, cr, p);
s_end = s + n;
last_p = NULL;
if (o != NULL && *o != UC_BREAK_UNDEFINED)
*p = *o;
- if (*p == UC_BREAK_POSSIBLE || *p == UC_BREAK_MANDATORY)
+ if (*p == UC_BREAK_POSSIBLE
+ || *p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF)
{
/* An atomic piece of text ends here. */
if (last_p != NULL && last_column + piece_width > width)
}
}
- if (*p == UC_BREAK_MANDATORY)
+ if (*p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF)
{
/* uc is a line break character. */
/* Start a new piece at column 0. */
return last_column + piece_width;
}
+
+#undef u32_width_linebreaks
+
+int
+u32_width_linebreaks (const uint32_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return u32_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, -1, p);
+}
+
+int
+u32_width_linebreaks_v2 (const uint32_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return u32_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, LBP_CR, p);
+}
/* Specification. */
#include "unilbrk.h"
+#include "unilbrk/internal.h"
#include <stdlib.h>
#include <string.h>
#include "unistr.h"
void
-u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding, char *p)
+u8_possible_linebreaks_loop (const uint8_t *s, size_t n, const char *encoding,
+ int cr, char *p)
{
if (n > 0)
{
int count = u8_mbtouc_unsafe (&uc, s, s_end - s);
int prop = unilbrkprop_lookup (uc);
- if (prop == LBP_BK)
+ if (prop == LBP_BK || prop == LBP_LF || prop == LBP_CR)
{
/* (LB4,LB5,LB6) Mandatory break. */
*p = UC_BREAK_MANDATORY;
- prev_prop = LBP_BK;
+ /* cr is either LBP_CR or -1. In the first case, recognize
+ a CR-LF sequence. */
+ if (prev_prop == cr && prop == LBP_LF)
+ p[-1] = UC_BREAK_CR_BEFORE_LF;
+ prev_prop = prop;
last_prop = LBP_BK;
seen_space = NULL;
}
}
}
+#undef u8_possible_linebreaks
+
+void
+u8_possible_linebreaks (const uint8_t *s, size_t n, const char *encoding,
+ char *p)
+{
+ u8_possible_linebreaks_loop (s, n, encoding, -1, p);
+}
+
+void
+u8_possible_linebreaks_v2 (const uint8_t *s, size_t n, const char *encoding,
+ char *p)
+{
+ u8_possible_linebreaks_loop (s, n, encoding, LBP_CR, p);
+}
+
#ifdef TEST
char *breaks = malloc (length);
int i;
- u8_possible_linebreaks ((uint8_t *) input, length, "UTF-8", breaks);
+ u8_possible_linebreaks_v2 ((uint8_t *) input, length, "UTF-8", breaks);
for (i = 0; i < length; i++)
{
/* U+21B2 (or U+21B5) in UTF-8 encoding */
putc (0xe2, stdout); putc (0x86, stdout); putc (0xb2, stdout);
break;
+ case UC_BREAK_CR_BEFORE_LF:
+ /* U+21E4 in UTF-8 encoding */
+ putc (0xe2, stdout); putc (0x87, stdout); putc (0xa4, stdout);
+ break;
case UC_BREAK_PROHIBITED:
break;
default:
/* Specification. */
#include "unilbrk.h"
+#include "unilbrk/internal.h"
+#include "unilbrk/lbrktables.h"
#include "unistr.h"
#include "uniwidth.h"
int
-u8_width_linebreaks (const uint8_t *s, size_t n,
- int width, int start_column, int at_end_columns,
- const char *o, const char *encoding,
- char *p)
+u8_width_linebreaks_internal (const uint8_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding, int cr,
+ char *p)
{
const uint8_t *s_end;
char *last_p;
int last_column;
int piece_width;
- u8_possible_linebreaks (s, n, encoding, p);
+ u8_possible_linebreaks_loop (s, n, encoding, cr, p);
s_end = s + n;
last_p = NULL;
if (o != NULL && *o != UC_BREAK_UNDEFINED)
*p = *o;
- if (*p == UC_BREAK_POSSIBLE || *p == UC_BREAK_MANDATORY)
+ if (*p == UC_BREAK_POSSIBLE
+ || *p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF)
{
/* An atomic piece of text ends here. */
if (last_p != NULL && last_column + piece_width > width)
}
}
- if (*p == UC_BREAK_MANDATORY)
+ if (*p == UC_BREAK_MANDATORY || *p == UC_BREAK_CR_BEFORE_LF)
{
/* uc is a line break character. */
/* Start a new piece at column 0. */
return last_column + piece_width;
}
+#undef u8_width_linebreaks
+
+int
+u8_width_linebreaks (const uint8_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return u8_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, -1, p);
+}
+
+int
+u8_width_linebreaks_v2 (const uint8_t *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return u8_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, LBP_CR, p);
+}
+
#ifdef TEST
char *breaks = malloc (length);
int i;
- u8_width_linebreaks ((uint8_t *) input, length, width, 0, 0, NULL, "UTF-8", breaks);
+ u8_width_linebreaks_v2 ((uint8_t *) input, length, width, 0, 0, NULL, "UTF-8", breaks);
for (i = 0; i < length; i++)
{
break;
case UC_BREAK_MANDATORY:
break;
+ case UC_BREAK_CR_BEFORE_LF:
+ break;
case UC_BREAK_PROHIBITED:
break;
default:
#include "c-ctype.h"
#include "uniconv.h"
+#include "unilbrk/internal.h"
+#include "unilbrk/lbrktables.h"
#include "unilbrk/ulc-common.h"
/* Line breaking of a string in an arbitrary encoding.
but this is not backed by an RFC. So we use UTF-8. It supports
characters up to \U7FFFFFFF and is unambiguously defined. */
-void
-ulc_possible_linebreaks (const char *s, size_t n, const char *encoding,
- char *p)
+static void
+ulc_possible_linebreaks_internal (const char *s, size_t n, const char *encoding,
+ int cr, char *p)
{
if (n > 0)
{
if (is_utf8_encoding (encoding))
- u8_possible_linebreaks ((const uint8_t *) s, n, encoding, p);
+ u8_possible_linebreaks_loop ((const uint8_t *) s, n, encoding, cr, p);
else
{
/* Convert the string to UTF-8 and build a translation table
/* Determine the possible line breaks of the UTF-8
string. */
- u8_possible_linebreaks (t, m, encoding, q);
+ u8_possible_linebreaks_loop (t, m, encoding, cr, q);
/* Translate the result back to the original string. */
memset (p, UC_BREAK_PROHIBITED, n);
if (is_all_ascii (s, n))
{
/* ASCII is a subset of UTF-8. */
- u8_possible_linebreaks ((const uint8_t *) s, n, encoding, p);
+ u8_possible_linebreaks_loop ((const uint8_t *) s, n, encoding, cr, p);
return;
}
#endif
const char *s_end = s + n;
while (s < s_end)
{
- *p = (*s == '\n' ? UC_BREAK_MANDATORY : UC_BREAK_PROHIBITED);
+ *p = (*s == '\n'
+ ? UC_BREAK_MANDATORY
+ : ((cr >= 0
+ && *s == '\r'
+ && s + 1 < s_end
+ && *(s + 1) == '\n')
+ ? UC_BREAK_CR_BEFORE_LF
+ : UC_BREAK_PROHIBITED));
s++;
p++;
}
}
}
+#undef ulc_possible_linebreaks
+
+void
+ulc_possible_linebreaks (const char *s, size_t n, const char *encoding,
+ char *p)
+{
+ ulc_possible_linebreaks_internal (s, n, encoding, -1, p);
+}
+
+void
+ulc_possible_linebreaks_v2 (const char *s, size_t n, const char *encoding,
+ char *p)
+{
+ ulc_possible_linebreaks_internal (s, n, encoding, LBP_CR, p);
+}
+
#ifdef TEST
char *breaks = malloc (length);
int i;
- ulc_possible_linebreaks (input, length, locale_charset (), breaks);
+ ulc_possible_linebreaks_v2 (input, length, locale_charset (), breaks);
for (i = 0; i < length; i++)
{
break;
case UC_BREAK_MANDATORY:
break;
+ case UC_BREAK_CR_BEFORE_LF:
+ break;
case UC_BREAK_PROHIBITED:
break;
default:
#include "c-ctype.h"
#include "uniconv.h"
+#include "unilbrk/internal.h"
+#include "unilbrk/lbrktables.h"
#include "unilbrk/ulc-common.h"
/* Line breaking of a string in an arbitrary encoding.
but this is not backed by an RFC. So we use UTF-8. It supports
characters up to \U7FFFFFFF and is unambiguously defined. */
-int
-ulc_width_linebreaks (const char *s, size_t n,
- int width, int start_column, int at_end_columns,
- const char *o, const char *encoding,
- char *p)
+static int
+ulc_width_linebreaks_internal (const char *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding, int cr,
+ char *p)
{
if (n > 0)
{
if (is_utf8_encoding (encoding))
- return u8_width_linebreaks ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, p);
+ return u8_width_linebreaks_internal ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, cr, p);
else
{
/* Convert the string to UTF-8 and build a translation table
/* Determine the line breaks of the UTF-8 string. */
res_column =
- u8_width_linebreaks (t, m, width, start_column, at_end_columns, o8, encoding, q);
+ u8_width_linebreaks_internal (t, m, width, start_column, at_end_columns, o8, encoding, cr, q);
/* Translate the result back to the original string. */
memset (p, UC_BREAK_PROHIBITED, n);
if (is_all_ascii (s, n))
{
/* ASCII is a subset of UTF-8. */
- return u8_width_linebreaks ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, p);
+ return u8_width_linebreaks_internal ((const uint8_t *) s, n, width, start_column, at_end_columns, o, encoding, cr, p);
}
#endif
/* We have a non-ASCII string and cannot convert it.
const char *s_end = s + n;
while (s < s_end)
{
- *p = ((o != NULL && *o == UC_BREAK_MANDATORY) || *s == '\n'
+ *p = ((o != NULL && *o == UC_BREAK_MANDATORY)
+ || *s == '\n'
? UC_BREAK_MANDATORY
- : UC_BREAK_PROHIBITED);
+ : ((o != NULL && *o == UC_BREAK_CR_BEFORE_LF)
+ || (cr >= 0
+ && *s == '\r'
+ && s + 1 < s_end
+ && *(s + 1) == '\n')
+ ? UC_BREAK_CR_BEFORE_LF
+ : UC_BREAK_PROHIBITED));
s++;
p++;
if (o != NULL)
return start_column;
}
+#undef ulc_width_linebreaks
+
+int
+ulc_width_linebreaks (const char *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return ulc_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, -1, p);
+}
+
+int
+ulc_width_linebreaks_v2 (const char *s, size_t n,
+ int width, int start_column, int at_end_columns,
+ const char *o, const char *encoding,
+ char *p)
+{
+ return ulc_width_linebreaks_internal (s, n,
+ width, start_column, at_end_columns,
+ o, encoding, LBP_CR, p);
+}
+
#ifdef TEST
char *breaks = malloc (length);
int i;
- ulc_width_linebreaks (input, length, width, 0, 0, NULL, locale_charset (), breaks);
+ ulc_width_linebreaks_v2 (input, length, width, 0, 0, NULL, locale_charset (), breaks);
for (i = 0; i < length; i++)
{
break;
case UC_BREAK_MANDATORY:
break;
+ case UC_BREAK_CR_BEFORE_LF:
+ break;
case UC_BREAK_PROHIBITED:
break;
default:
Line breaking of UTF-16 strings.
Files:
+lib/unilbrk/internal.h
lib/unilbrk/u16-possible-linebreaks.c
lib/uniwidth/cjk.h
Line breaking of UTF-32 strings.
Files:
+lib/unilbrk/internal.h
lib/unilbrk/u32-possible-linebreaks.c
lib/uniwidth/cjk.h
Line breaking of UTF-8 strings.
Files:
+lib/unilbrk/internal.h
lib/unilbrk/u8-possible-linebreaks.c
lib/uniwidth/cjk.h
Line breaking of UTF-8 strings.
Files:
+lib/unilbrk/internal.h
lib/unilbrk/u8-width-linebreaks.c
Depends-on:
#include "macros.h"
-int
-main ()
+static void
+test_function (void (*my_u16_possible_linebreaks) (const uint16_t *, size_t, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- u16_possible_linebreaks (NULL, 0, "GB18030", NULL);
+ my_u16_possible_linebreaks (NULL, 0, "GB18030", NULL);
{
static const uint16_t input[61] =
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "GB18030", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "GB18030", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "GB2312", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "GB2312", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
}
}
+ /* CR LF handling. */
+ {
+ static const uint16_t input[8] =
+ { 'a', '\n', 'b', '\r', 'c', '\r', '\n', 'd' };
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 8; i++)
+ {
+ ASSERT (p[i] == (i == 1 || i == 3 || i == 6 ? UC_BREAK_MANDATORY :
+ i == 5 ? (version < 2 ? UC_BREAK_MANDATORY : UC_BREAK_CR_BEFORE_LF) :
+ UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
/* Test that a break is possible after a zero-width space followed by some
regular spaces (rule LB8 in Unicode TR#14 revision 26). */
{
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 4; i++)
{
ASSERT (p[i] == (i == 3 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 21; i++)
{
ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 16; i++)
{
ASSERT (p[i] == (i == 14 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 39; i++)
{
ASSERT (p[i] == (i == 8 || i == 20
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 8; i++)
{
ASSERT (p[i] == (i == 4 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u16_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 10; i++)
{
ASSERT (p[i] == (i == 3 || i == 9 ? UC_BREAK_POSSIBLE :
}
free (p);
}
+}
+
+int
+main ()
+{
+ test_function (u16_possible_linebreaks, 2);
+#undef u16_possible_linebreaks
+ test_function (u16_possible_linebreaks, 1);
return 0;
}
#include "macros.h"
-int
-main ()
+static void
+test_function (int (*my_u16_width_linebreaks) (const uint16_t *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- u16_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
+ my_u16_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
{
static const uint16_t input[61] =
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p);
+ my_u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p);
+ my_u16_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
free (p);
}
}
+}
+
+int
+main ()
+{
+ test_function (u16_width_linebreaks, 2);
+#undef u16_width_linebreaks
+ test_function (u16_width_linebreaks, 1);
return 0;
}
#include "macros.h"
-int
-main ()
+static void
+test_function (void (*my_u32_possible_linebreaks) (const uint32_t *, size_t, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- u32_possible_linebreaks (NULL, 0, "GB18030", NULL);
+ my_u32_possible_linebreaks (NULL, 0, "GB18030", NULL);
{
static const uint32_t input[61] =
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "GB18030", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "GB18030", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "GB2312", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "GB2312", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
}
}
+ /* CR LF handling. */
+ {
+ static const uint32_t input[8] =
+ { 'a', '\n', 'b', '\r', 'c', '\r', '\n', 'd' };
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 8; i++)
+ {
+ ASSERT (p[i] == (i == 1 || i == 3 || i == 6 ? UC_BREAK_MANDATORY :
+ i == 5 ? (version < 2 ? UC_BREAK_MANDATORY : UC_BREAK_CR_BEFORE_LF) :
+ UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
/* Test that a break is possible after a zero-width space followed by some
regular spaces (rule LB8 in Unicode TR#14 revision 26). */
{
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 4; i++)
{
ASSERT (p[i] == (i == 3 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 21; i++)
{
ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 16; i++)
{
ASSERT (p[i] == (i == 14 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 37; i++)
{
ASSERT (p[i] == (i == 8 || i == 20
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 4; i++)
{
ASSERT (p[i] == (i == 2 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u32_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 10; i++)
{
ASSERT (p[i] == (i == 3 || i == 9 ? UC_BREAK_POSSIBLE :
}
free (p);
}
+}
+
+int
+main ()
+{
+ test_function (u32_possible_linebreaks, 2);
+#undef u32_possible_linebreaks
+ test_function (u32_possible_linebreaks, 1);
return 0;
}
#include "macros.h"
-int
-main ()
+static void
+test_function (int (*my_u32_width_linebreaks) (const uint32_t *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- u32_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
+ my_u32_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
{
static const uint32_t input[61] =
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p);
+ my_u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p);
+ my_u32_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p);
for (i = 0; i < 61; i++)
{
ASSERT (p[i] == (i == 60 ? UC_BREAK_MANDATORY :
free (p);
}
}
+}
+
+int
+main ()
+{
+ test_function (u32_width_linebreaks, 2);
+#undef u32_width_linebreaks
+ test_function (u32_width_linebreaks, 1);
return 0;
}
#include "macros.h"
-int
-main ()
+static void
+test_function (void (*my_u8_possible_linebreaks) (const uint8_t *, size_t, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- u8_possible_linebreaks (NULL, 0, "GB18030", NULL);
+ my_u8_possible_linebreaks (NULL, 0, "GB18030", NULL);
{
static const uint8_t input[91] =
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "GB18030", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "GB18030", p);
for (i = 0; i < 91; i++)
{
ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "GB2312", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "GB2312", p);
for (i = 0; i < 91; i++)
{
ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY :
}
}
+ /* CR LF handling. */
+ {
+ static const uint8_t input[8] = "a\nb\rc\r\nd";
+ char *p = (char *) malloc (SIZEOF (input));
+ size_t i;
+
+ my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ for (i = 0; i < 8; i++)
+ {
+ ASSERT (p[i] == (i == 1 || i == 3 || i == 6 ? UC_BREAK_MANDATORY :
+ i == 5 ? (version < 2 ? UC_BREAK_MANDATORY : UC_BREAK_CR_BEFORE_LF) :
+ UC_BREAK_PROHIBITED));
+ }
+ free (p);
+ }
+
/* Test that a break is possible after a zero-width space followed by some
regular spaces (rule LB8 in Unicode TR#14 revision 26). */
{
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 6; i++)
{
ASSERT (p[i] == (i == 5 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 21; i++)
{
ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 24; i++)
{
ASSERT (p[i] == (i == 21 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 101; i++)
{
ASSERT (p[i] == (i == 24 || i == 58
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 16; i++)
{
ASSERT (p[i] == (i == 8 ? UC_BREAK_POSSIBLE : UC_BREAK_PROHIBITED));
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_u8_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 15; i++)
{
ASSERT (p[i] == (i == 3 || i == 13 ? UC_BREAK_POSSIBLE :
}
free (p);
}
+}
+
+int
+main ()
+{
+ test_function (u8_possible_linebreaks, 2);
+#undef u8_possible_linebreaks
+ test_function (u8_possible_linebreaks, 1);
return 0;
}
#include "macros.h"
-int
-main ()
+static void
+test_function (int (*my_u8_width_linebreaks) (const uint8_t *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- u8_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
+ my_u8_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
{
static const uint8_t input[91] =
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p);
+ my_u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB18030", p);
for (i = 0; i < 91; i++)
{
ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p);
+ my_u8_width_linebreaks (input, SIZEOF (input), 25, 0, 0, NULL, "GB2312", p);
for (i = 0; i < 91; i++)
{
ASSERT (p[i] == (i == 90 ? UC_BREAK_MANDATORY :
free (p);
}
}
+}
+
+int
+main ()
+{
+ test_function (u8_width_linebreaks, 2);
+#undef u8_width_linebreaks
+ test_function (u8_width_linebreaks, 1);
return 0;
}
#include "macros.h"
-int
-main ()
+static void
+test_function (void (*my_ulc_possible_linebreaks) (const char *, size_t, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- ulc_possible_linebreaks (NULL, 0, "GB18030", NULL);
+ my_ulc_possible_linebreaks (NULL, 0, "GB18030", NULL);
#if HAVE_ICONV
{
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- ulc_possible_linebreaks (input, SIZEOF (input), "ISO-8859-1", p);
+ my_ulc_possible_linebreaks (input, SIZEOF (input), "ISO-8859-1", p);
for (i = 0; i < 36; i++)
{
ASSERT (p[i] == (i == 35 ? UC_BREAK_MANDATORY :
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- ulc_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
+ my_ulc_possible_linebreaks (input, SIZEOF (input), "UTF-8", p);
for (i = 0; i < 21; i++)
{
ASSERT (p[i] == (i == 8 || i == 17 || i == 19 ? UC_BREAK_POSSIBLE :
free (p);
}
#endif
+}
+
+int
+main ()
+{
+ test_function (ulc_possible_linebreaks, 2);
+#undef ulc_possible_linebreaks
+ test_function (ulc_possible_linebreaks, 1);
return 0;
}
#include "macros.h"
-int
-main ()
+static void
+test_function (int (*my_ulc_width_linebreaks) (const char *, size_t, int, int, int, const char *, const char *, char *_UC_RESTRICT),
+ int version)
{
/* Test case n = 0. */
- ulc_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
+ my_ulc_width_linebreaks (NULL, 0, 80, 0, 0, NULL, "GB18030", NULL);
#if HAVE_ICONV
{
char *p = (char *) malloc (SIZEOF (input));
size_t i;
- ulc_width_linebreaks (input, SIZEOF (input), 12, 0, 0, NULL, "ISO-8859-1", p);
+ my_ulc_width_linebreaks (input, SIZEOF (input), 12, 0, 0, NULL, "ISO-8859-1", p);
for (i = 0; i < 36; i++)
{
ASSERT (p[i] == (i == 35 ? UC_BREAK_MANDATORY :
free (p);
}
#endif
+}
+
+int
+main ()
+{
+ test_function (ulc_width_linebreaks, 2);
+#undef ulc_width_linebreaks
+ test_function (ulc_width_linebreaks, 1);
return 0;
}