From: Bruno Haible Date: Sun, 16 Jul 2023 05:30:15 +0000 (+0200) Subject: mbszero: New module. X-Git-Tag: v1.0~1107 X-Git-Url: https://gitweb.git.savannah.gnu.org/gitweb/?a=commitdiff_plain;h=c96c9e2d50f46edcd4e56b821bc469949c0dc14e;p=gnulib.git mbszero: New module. * lib/wchar.in.h: Include . (_GL_MBSTATE_INIT_SIZE, _GL_MBSTATE_ZERO_SIZE): New macros. (mbszero): New declaration. * lib/mbrtoc16.c: Update comments. * lib/mbszero.c: New file. * m4/wchar_h.m4 (gl_WCHAR_H_REQUIRE_DEFAULTS): Initialize GNULIB_MBSZERO. * modules/wchar (Depends-on): Add extern-inline. (Makefile.am): Substitute GNULIB_MBSZERO. * modules/mbszero: New file. --- diff --git a/ChangeLog b/ChangeLog index 68235627fb..151d9b4537 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +2023-07-16 Bruno Haible + + mbszero: New module. + * lib/wchar.in.h: Include . + (_GL_MBSTATE_INIT_SIZE, _GL_MBSTATE_ZERO_SIZE): New macros. + (mbszero): New declaration. + * lib/mbrtoc16.c: Update comments. + * lib/mbszero.c: New file. + * m4/wchar_h.m4 (gl_WCHAR_H_REQUIRE_DEFAULTS): Initialize + GNULIB_MBSZERO. + * modules/wchar (Depends-on): Add extern-inline. + (Makefile.am): Substitute GNULIB_MBSZERO. + * modules/mbszero: New file. + 2023-07-15 Bruno Haible mbsinit: Fix module description. diff --git a/lib/mbrtoc16.c b/lib/mbrtoc16.c index eb73e7d447..1afcde44cc 100644 --- a/lib/mbrtoc16.c +++ b/lib/mbrtoc16.c @@ -54,24 +54,26 @@ static_assert (sizeof (mbstate_t) >= 4); /* macOS, FreeBSD, NetBSD, OpenBSD, Minix */ /* On macOS, mbstate_t is defined in . It is an opaque aligned 128-byte struct, of which at most the first - 20 bytes are used (the members are at most: 2x wchar_t, 2x int, 4x char). + 12 bytes are used. For more details, see the __mbsinit implementations in Libc-/locale/FreeBSD/ {ascii,none,euc,mskanji,big5,gb2312,gbk,gb18030,utf8,utf2}.c. */ /* On FreeBSD, mbstate_t is defined in src/sys/sys/_types.h. It is an opaque aligned 128-byte struct, of which at most the first - 20 bytes are used (the members are at most: 2x wchar_t, 2x int, 4x char). + 12 bytes are used. For more details, see the __mbsinit implementations in src/lib/libc/locale/ {ascii,none,euc,mskanji,big5,gb2312,gbk,gb18030,utf8}.c. */ /* On NetBSD, mbstate_t is defined in src/sys/sys/ansi.h. It is an opaque aligned 128-byte struct, of which at most the first - 24 bytes are used (the members are at most: 3x int, 12x char). + 28 bytes are used. For more details, see the *State types in - src/lib/libc/citrus/modules/citrus_*.c. */ + src/lib/libc/citrus/modules/citrus_*.c + (ignoring citrus_{hz,iso2022,utf7,viqr,zw}.c, since these implement + stateful encodings, not usable as locale encodings). */ /* On OpenBSD, mbstate_t is defined in src/sys/sys/_types.h. It is an opaque aligned 128-byte struct, of which at most the first - 12 bytes are used (the members are at most: 2x wchar_t, 1x int). + 12 bytes are used. For more details, see src/lib/libc/citrus/citrus_*.c. */ /* Minix has borrowed its mbstate_t type and mbrtowc implementation from the BSDs. */ @@ -84,7 +86,7 @@ static_assert (sizeof (mbstate_t) >= 4); #elif defined __sun /* Solaris */ /* On Solaris, mbstate_t is defined in . It is an opaque aligned 24-byte or 32-byte struct, of which at most the first - 20 bytes are used (the members are at most: 2x wchar_t, 2x int, 4x char). + 20 or 28 bytes are used. For more details, see the *State types in illumos-gate/usr/src/lib/libc/port/locale/ {none,euc,mskanji,big5,gb2312,gbk,gb18030,utf8}.c. */ diff --git a/lib/mbszero.c b/lib/mbszero.c new file mode 100644 index 0000000000..6da91c6b20 --- /dev/null +++ b/lib/mbszero.c @@ -0,0 +1,23 @@ +/* Put an mbstate_t into an initial conversion state. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; either version 2.1 of the + License, or (at your option) any later version. + + This file is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program. If not, see . */ + +/* Written by Bruno Haible , 2023. */ + +#include + +#define IN_MBSZERO +/* Specification and implementation. */ +#include diff --git a/lib/wchar.in.h b/lib/wchar.in.h index 2c878ff815..7d2c5ecd12 100644 --- a/lib/wchar.in.h +++ b/lib/wchar.in.h @@ -232,6 +232,13 @@ _GL_EXTERN_C void free (void *); # endif #endif + +#if @GNULIB_MBSZERO@ +/* Get memset(). */ +# include +#endif + + /* Convert a single-byte character to a wide character. */ #if @GNULIB_BTOWC@ # if @REPLACE_BTOWC@ @@ -315,6 +322,203 @@ _GL_WARN_ON_USE (mbsinit, "mbsinit is unportable - " #endif +/* Put *PS into an initial state. */ +#if @GNULIB_MBSZERO@ +/* ISO C 23 § 7.31.6.(3) says that zeroing an mbstate_t is a way to put the + mbstate_t into an initial state. However, on many platforms an mbstate_t + is large, and it is possible - as an optimization - to get away with zeroing + only part of it. So, instead of + + mbstate_t state = { 0 }; + + or + + mbstate_t state; + memset (&state, 0, sizeof (mbstate_t)); + + we can write this faster code: + + mbstate_t state; + mbszero (&state); + */ +/* _GL_MBSTATE_INIT_SIZE describes how mbsinit() behaves: It is the number of + bytes at the beginning of an mbstate_t that need to be zero, for mbsinit() + to return true. + _GL_MBSTATE_ZERO_SIZE is the number of bytes at the beginning of an mbstate_t + that need to be zero, + - for mbsinit() to return true, and + - for all other multibyte-aware functions to operate properly. + 0 < _GL_MBSTATE_INIT_SIZE <= _GL_MBSTATE_ZERO_SIZE <= sizeof (mbstate_t). + These values are determined by source code inspection. */ +# if GNULIB_defined_mbstate_t /* AIX, IRIX */ +/* mbstate_t has at least 4 bytes. They are used as coded in + gnulib/lib/mbrtowc.c. */ +# define _GL_MBSTATE_INIT_SIZE 1 +/* Note that 4 is not the correct value: it causes test failures. */ +# define _GL_MBSTATE_ZERO_SIZE sizeof (mbstate_t) +# elif __GLIBC__ >= 2 /* glibc */ +/* mbstate_t is defined in . + For more details, see glibc/iconv/skeleton.c. */ +# define _GL_MBSTATE_INIT_SIZE 4 +# define _GL_MBSTATE_ZERO_SIZE /* 8 */ sizeof (mbstate_t) +# elif defined MUSL_LIBC /* musl libc */ +/* mbstate_t is defined in . + It is an opaque aligned 8-byte struct, of which at most the first + 4 bytes are used. + For more details, see src/multibyte/mbrtowc.c. */ +# define _GL_MBSTATE_INIT_SIZE 4 +# define _GL_MBSTATE_ZERO_SIZE 4 +# elif defined __APPLE__ && defined __MACH__ /* macOS */ +/* On macOS, mbstate_t is defined in . + It is an opaque aligned 128-byte struct, of which at most the first + 12 bytes are used. + For more details, see the __mbsinit implementations in + Libc-/locale/FreeBSD/ + {ascii,none,euc,mskanji,big5,gb2312,gbk,gb18030,utf8,utf2}.c. */ +/* File INIT_SIZE ZERO_SIZE + ascii.c 0 0 + none.c 0 0 + euc.c 12 12 + mskanji.c 4 4 + big5.c 4 4 + gb2312.c 4 6 + gbk.c 4 4 + gb18030.c 4 8 + utf8.c 8 10 + utf2.c 8 12 */ +# define _GL_MBSTATE_INIT_SIZE 12 +# define _GL_MBSTATE_ZERO_SIZE 12 +# elif defined __FreeBSD__ /* FreeBSD */ +/* On FreeBSD, mbstate_t is defined in src/sys/sys/_types.h. + It is an opaque aligned 128-byte struct, of which at most the first + 12 bytes are used. + For more details, see the __mbsinit implementations in + src/lib/libc/locale/ + {ascii,none,euc,mskanji,big5,gb2312,gbk,gb18030,utf8}.c. */ +/* File INIT_SIZE ZERO_SIZE + ascii.c 0 0 + none.c 0 0 + euc.c 12 12 + mskanji.c 4 4 + big5.c 4 4 + gb2312.c 4 6 + gbk.c 4 4 + gb18030.c 4 8 + utf8.c 8 12 */ +# define _GL_MBSTATE_INIT_SIZE 12 +# define _GL_MBSTATE_ZERO_SIZE 12 +# elif defined __NetBSD__ /* NetBSD */ +/* On NetBSD, mbstate_t is defined in src/sys/sys/ansi.h. + It is an opaque aligned 128-byte struct, of which at most the first + 28 bytes are used. + For more details, see the *State types in + src/lib/libc/citrus/modules/citrus_*.c + (ignoring citrus_{hz,iso2022,utf7,viqr,zw}.c, since these implement + stateful encodings, not usable as locale encodings). */ +/* File ZERO_SIZE + citrus/citrus_none.c 0 + citrus/modules/citrus_euc.c 8 + citrus/modules/citrus_euctw.c 8 + citrus/modules/citrus_mskanji.c 8 + citrus/modules/citrus_big5.c 8 + citrus/modules/citrus_gbk2k.c 8 + citrus/modules/citrus_dechanyu.c 8 + citrus/modules/citrus_johab.c 6 + citrus/modules/citrus_utf8.c 12 */ +/* But 12 is not the correct value: we get test failures for values < 28. */ +# define _GL_MBSTATE_INIT_SIZE 28 +# define _GL_MBSTATE_ZERO_SIZE 28 +# elif defined __OpenBSD__ /* OpenBSD */ +/* On OpenBSD, mbstate_t is defined in src/sys/sys/_types.h. + It is an opaque aligned 128-byte struct, of which at most the first + 12 bytes are used. + For more details, see src/lib/libc/citrus/citrus_*.c. */ +/* File INIT_SIZE ZERO_SIZE + citrus_none.c 0 0 + citrus_utf8.c 12 12 */ +# define _GL_MBSTATE_INIT_SIZE 12 +# define _GL_MBSTATE_ZERO_SIZE 12 +# elif defined __minix /* Minix */ +/* On Minix, mbstate_t is defined in sys/sys/ansi.h. + It is an opaque aligned 128-byte struct. + For more details, see the *State types in + lib/libc/citrus/citrus_*.c. */ +/* File INIT_SIZE ZERO_SIZE + citrus_none.c 0 0 */ +# define _GL_MBSTATE_INIT_SIZE 1 +# define _GL_MBSTATE_ZERO_SIZE 1 +# elif defined __sun /* Solaris */ +/* On Solaris, mbstate_t is defined in . + It is an opaque aligned 24-byte or 32-byte struct, of which at most the first + 20 or 28 bytes are used. + For more details, see the *State types in + illumos-gate/usr/src/lib/libc/port/locale/ + {none,euc,mskanji,big5,gb2312,gbk,gb18030,utf8}.c. */ +/* File INIT_SIZE ZERO_SIZE + none.c 0 0 + euc.c 12 12 + mskanji.c 4 4 + big5.c 4 4 + gb2312.c 4 6 + gbk.c 4 4 + gb18030.c 4 8 + utf8.c 12 12 */ +/* But 12 is not the correct value: we get test failures + - in OpenIndiana and OmniOS: for values < 16, + - in Solaris 10 and 11: for values < 20 (in 32-bit mode) + or < 28 (in 64-bit mode). */ +# if defined _LP64 +# define _GL_MBSTATE_INIT_SIZE 28 +# define _GL_MBSTATE_ZERO_SIZE 28 +# else +# define _GL_MBSTATE_INIT_SIZE 20 +# define _GL_MBSTATE_ZERO_SIZE 20 +# endif +# elif defined __CYGWIN__ /* Cygwin */ +/* On Cygwin, mbstate_t is defined in . + For more details, see newlib/libc/stdlib/mbtowc_r.c and + winsup/cygwin/strfuncs.cc. */ +# define _GL_MBSTATE_INIT_SIZE 4 +# define _GL_MBSTATE_ZERO_SIZE 8 +# elif defined _WIN32 && !defined __CYGWIN__ /* Native Windows. */ +/* MSVC defines 'mbstate_t' as an aligned 8-byte struct. + On mingw, 'mbstate_t' is sometimes defined as 'int', sometimes defined + as an aligned 8-byte struct, of which the first 4 bytes matter. */ +# define _GL_MBSTATE_INIT_SIZE sizeof (mbstate_t) +# define _GL_MBSTATE_ZERO_SIZE sizeof (mbstate_t) +# elif defined __ANDROID__ /* Android */ +/* Android defines 'mbstate_t' in . + It is an opaque 4-byte or 8-byte struct. + For more details, see + bionic/libc/private/bionic_mbstate.h + bionic/libc/bionic/mbrtoc32.cpp + bionic/libc/bionic/mbrtoc16.cpp + */ +# define _GL_MBSTATE_INIT_SIZE 4 +# define _GL_MBSTATE_ZERO_SIZE 4 +# else +/* On platforms where we don't know how the multibyte functions behave, use + these safe values. */ +# define _GL_MBSTATE_INIT_SIZE sizeof (mbstate_t) +# define _GL_MBSTATE_ZERO_SIZE sizeof (mbstate_t) +# endif +_GL_BEGIN_C_LINKAGE +# if defined IN_MBSZERO +_GL_EXTERN_INLINE +# else +_GL_INLINE +# endif +_GL_ARG_NONNULL ((1)) void +mbszero (mbstate_t *ps) +{ + memset (ps, 0, _GL_MBSTATE_ZERO_SIZE); +} +_GL_END_C_LINKAGE +_GL_CXXALIAS_SYS (mbszero, void, (mbstate_t *ps)); +_GL_CXXALIASWARN (mbszero); +#endif + + /* Convert a multibyte character to a wide character. */ #if @GNULIB_MBRTOWC@ # if @REPLACE_MBRTOWC@ diff --git a/m4/wchar_h.m4 b/m4/wchar_h.m4 index 442932be44..31f5b0794d 100644 --- a/m4/wchar_h.m4 +++ b/m4/wchar_h.m4 @@ -7,7 +7,7 @@ dnl with or without modifications, as long as this notice is preserved. dnl Written by Eric Blake. -# wchar_h.m4 serial 60 +# wchar_h.m4 serial 61 AC_DEFUN_ONCE([gl_WCHAR_H], [ @@ -147,6 +147,7 @@ AC_DEFUN([gl_WCHAR_H_REQUIRE_DEFAULTS], gl_MODULE_INDICATOR_INIT_VARIABLE([GNULIB_BTOWC]) gl_MODULE_INDICATOR_INIT_VARIABLE([GNULIB_WCTOB]) gl_MODULE_INDICATOR_INIT_VARIABLE([GNULIB_MBSINIT]) + gl_MODULE_INDICATOR_INIT_VARIABLE([GNULIB_MBSZERO]) gl_MODULE_INDICATOR_INIT_VARIABLE([GNULIB_MBRTOWC]) gl_MODULE_INDICATOR_INIT_VARIABLE([GNULIB_MBRLEN]) gl_MODULE_INDICATOR_INIT_VARIABLE([GNULIB_MBSRTOWCS]) diff --git a/modules/mbszero b/modules/mbszero new file mode 100644 index 0000000000..002fa75014 --- /dev/null +++ b/modules/mbszero @@ -0,0 +1,29 @@ +Description: +mbszero() function: put an mbstate_t into an initial conversion state. + +Files: +lib/mbszero.c +m4/mbstate_t.m4 +m4/mbrtowc.m4 +m4/musl.m4 + +Depends-on: +wchar + +configure.ac: +AC_REQUIRE([AC_TYPE_MBSTATE_T]) +gl_MBSTATE_T_BROKEN +gl_MUSL_LIBC +gl_WCHAR_MODULE_INDICATOR([mbszero]) + +Makefile.am: +lib_SOURCES += mbszero.c + +Include: + + +License: +LGPLv2+ + +Maintainer: +all diff --git a/modules/wchar b/modules/wchar index 6a17e53034..ebdd0ece61 100644 --- a/modules/wchar +++ b/modules/wchar @@ -13,6 +13,7 @@ include_next snippet/arg-nonnull snippet/c++defs snippet/warn-on-use +extern-inline inttypes-incomplete stddef stdlib @@ -42,6 +43,7 @@ wchar.h: wchar.in.h $(top_builddir)/config.status $(CXXDEFS_H) $(ARG_NONNULL_H) -e 's/@''GNULIB_BTOWC''@/$(GNULIB_BTOWC)/g' \ -e 's/@''GNULIB_WCTOB''@/$(GNULIB_WCTOB)/g' \ -e 's/@''GNULIB_MBSINIT''@/$(GNULIB_MBSINIT)/g' \ + -e 's/@''GNULIB_MBSZERO''@/$(GNULIB_MBSZERO)/g' \ -e 's/@''GNULIB_MBRTOWC''@/$(GNULIB_MBRTOWC)/g' \ -e 's/@''GNULIB_MBRLEN''@/$(GNULIB_MBRLEN)/g' \ -e 's/@''GNULIB_MBSRTOWCS''@/$(GNULIB_MBSRTOWCS)/g' \