* lib/localeinfo.h: Include <uchar.h>. Add special definitions for GAWK.
(case_folded_counterparts): Change array element type to char32_t.
* lib/localeinfo.c: Include <uchar.h>. Add special definitions for GAWK.
(is_using_utf8, init_localeinfo): Use mbrtoc32 instead of mbrtowc.
(lonesome_lower): Change element type to 'unsigned short'.
(case_folded_counterparts): Change array element type to char32_t. Use
c32toupper instead of towupper. Use c32tolower instead of towlower.
* lib/dfa.c: Include <uchar.h>. Add special definitions for GAWK.
(struct mb_char_classes): Change element type of 'chars' to char32_t.
(mbs_to_wchar): Use mbrtoc32 instead of mbrtowc.
(setbit_wc): Change type of first argument to char32_t. Use c32tob
instead of wctob.
(parse_bracket_exp): Update.
(lex): Use c32isprint instead of iswprint. Use c32isspace instead of
iswspace. Use c32rtomb instead of a %lc directive.
(addtok_wc): Use c32rtomb instead of wcrtomb.
(atom): Update.
* modules/dfa (Depends-on): Remove wctype-h. Add uchar,
mbrtoc32-regular, c32rtomb, c32tob, c32tolower, c32toupper, c32isprint,
c32isspace.
(Link): Add $(LIBUNISTRING) $(LIBC32CONV).
* modules/dfa-tests (Makefile.am): Link test-dfa-match-aux with
$(LIBUNISTRING) $(LIBC32CONV).
* NEWS: Mention the change.
+2023-07-10 Bruno Haible <bruno@clisp.org>
+
+ dfa: Overcome wchar_t limitations.
+ * lib/localeinfo.h: Include <uchar.h>. Add special definitions for GAWK.
+ (case_folded_counterparts): Change array element type to char32_t.
+ * lib/localeinfo.c: Include <uchar.h>. Add special definitions for GAWK.
+ (is_using_utf8, init_localeinfo): Use mbrtoc32 instead of mbrtowc.
+ (lonesome_lower): Change element type to 'unsigned short'.
+ (case_folded_counterparts): Change array element type to char32_t. Use
+ c32toupper instead of towupper. Use c32tolower instead of towlower.
+ * lib/dfa.c: Include <uchar.h>. Add special definitions for GAWK.
+ (struct mb_char_classes): Change element type of 'chars' to char32_t.
+ (mbs_to_wchar): Use mbrtoc32 instead of mbrtowc.
+ (setbit_wc): Change type of first argument to char32_t. Use c32tob
+ instead of wctob.
+ (parse_bracket_exp): Update.
+ (lex): Use c32isprint instead of iswprint. Use c32isspace instead of
+ iswspace. Use c32rtomb instead of a %lc directive.
+ (addtok_wc): Use c32rtomb instead of wcrtomb.
+ (atom): Update.
+ * modules/dfa (Depends-on): Remove wctype-h. Add uchar,
+ mbrtoc32-regular, c32rtomb, c32tob, c32tolower, c32toupper, c32isprint,
+ c32isspace.
+ (Link): Add $(LIBUNISTRING) $(LIBC32CONV).
+ * modules/dfa-tests (Makefile.am): Link test-dfa-match-aux with
+ $(LIBUNISTRING) $(LIBC32CONV).
+ * NEWS: Mention the change.
+
2023-07-10 Bruno Haible <bruno@clisp.org>
mbrtoc32-regular: Add tests.
Date Modules Changes
+2023-07-10 dfa The signature of the function
+ case_folded_counterparts, declared in localeinfo.h,
+ has changed.
+
2023-06-10 javacomp-script These modules now compile the Java code with option
javacomp '-source 1.6' or higher. As a consequence, the
compiler may emit notes "... uses unchecked or
#include <stdlib.h>
#include <limits.h>
#include <string.h>
+#include <wchar.h>
+
+#include "xalloc.h"
+#include "localeinfo.h"
+
+#include "gettext.h"
+#define _(str) gettext (str)
+
+#if GAWK
+/* Use ISO C 99 API. */
+# include <wctype.h>
+# define char32_t wchar_t
+# define mbrtoc32 mbrtowc
+# define c32rtomb wcrtomb
+# define c32tob wctob
+# define c32isprint iswprint
+# define c32isspace iswspace
+#else
+/* Use ISO C 11 + gnulib API. */
+# include <uchar.h>
+#endif
/* Pacify gcc -Wanalyzer-null-dereference in areas where GCC
understandably cannot deduce that the input comes from a
return '0' <= c && c <= '9';
}
-#include "gettext.h"
-#define _(str) gettext (str)
-
-#include <wchar.h>
-#include <wctype.h>
-
-#include "xalloc.h"
-#include "localeinfo.h"
-
#ifndef FALLTHROUGH
# if 201710L < __STDC_VERSION__
# define FALLTHROUGH [[__fallthrough__]]
RPAREN, /* RPAREN never appears in the parse tree. */
- WCHAR, /* Only returned by lex. wctok contains
- the wide character representation. */
+ WCHAR, /* Only returned by lex. wctok contains the
+ 32-bit wide character representation. */
ANYCHAR, /* ANYCHAR is a terminal symbol that matches
a valid multibyte (or single byte) character.
{
ptrdiff_t cset;
bool invert;
- wchar_t *chars; /* Normal characters. */
+ char32_t *chars; /* Normal characters. */
idx_t nchars;
idx_t nchars_alloc;
};
idx_t parens; /* Count of outstanding left parens. */
int minrep, maxrep; /* Repeat counts for {m,n}. */
- /* Wide character representation of the current multibyte character,
+ /* 32-bit wide character representation of the current multibyte character,
or WEOF if there was an encoding error. Used only if
MB_CUR_MAX > 1. */
wint_t wctok;
convert just a single byte, to WEOF. Return the number of bytes
converted.
- This differs from mbrtowc (PWC, S, N, &D->mbs) as follows:
+ This differs from mbrtoc32 (PWC, S, N, &D->mbs) as follows:
- * PWC points to wint_t, not to wchar_t.
+ * PWC points to wint_t, not to char32_t.
* The last arg is a dfa *D instead of merely a multibyte conversion
state D->mbs.
* N is idx_t not size_t, and must be at least 1.
if (wc == WEOF)
{
- wchar_t wch;
- size_t nbytes = mbrtowc (&wch, s, n, &d->mbs);
+ char32_t wch;
+ size_t nbytes = mbrtoc32 (&wch, s, n, &d->mbs);
if (0 < nbytes && nbytes < (size_t) -2)
{
*pwc = wch;
+ /* nbytes cannot be == (size) -3 here, since we rely on the
+ 'mbrtoc32-regular' module. */
return nbytes;
}
memset (&d->mbs, 0, sizeof d->mbs);
return CTX_NONE;
}
-/* Set a bit in the charclass for the given wchar_t. Do nothing if WC
+/* Set a bit in the charclass for the given char32_t. Do nothing if WC
is represented by a multi-byte sequence. Even for MB_CUR_MAX == 1,
this may happen when folding case in weird Turkish locales where
dotless i/dotted I are not included in the chosen character set.
Return whether a bit was set in the charclass. */
static bool
-setbit_wc (wint_t wc, charclass *c)
+setbit_wc (char32_t wc, charclass *c)
{
- int b = wctob (wc);
+ int b = c32tob (wc);
if (b < 0)
return false;
known_bracket_exp = false;
else
{
- wchar_t folded[CASE_FOLDED_BUFSIZE + 1];
+ char32_t folded[CASE_FOLDED_BUFSIZE + 1];
int n = (dfa->syntax.case_fold
? case_folded_counterparts (wc, folded + 1) + 1
: 1);
{
char const *msg;
char msgbuf[100];
- if (!iswprint (dfa->lex.wctok))
+ if (!c32isprint (dfa->lex.wctok))
msg = _("stray \\ before unprintable character");
- else if (iswspace (dfa->lex.wctok))
+ else if (c32isspace (dfa->lex.wctok))
msg = _("stray \\ before white space");
else
{
- int n = snprintf (msgbuf, sizeof msgbuf,
- _("stray \\ before %lc"), dfa->lex.wctok);
- msg = 0 <= n && n < sizeof msgbuf ? msgbuf : _("stray \\");
+ char buf[MB_LEN_MAX + 1];
+ mbstate_t s = { 0 };
+ size_t stored_bytes = c32rtomb (buf, dfa->lex.wctok, &s);
+ if (stored_bytes < (size_t) -1)
+ {
+ buf[stored_bytes] = '\0';
+ int n = snprintf (msgbuf, sizeof msgbuf,
+ _("stray \\ before %s"), buf);
+ msg = 0 <= n && n < sizeof msgbuf ? msgbuf : _("stray \\");
+ }
+ else
+ msg = _("stray \\");
}
dfawarn (msg);
}
{
unsigned char buf[MB_LEN_MAX];
mbstate_t s = { 0 };
- size_t stored_bytes = wcrtomb ((char *) buf, wc, &s);
+ size_t stored_bytes = c32rtomb ((char *) buf, wc, &s);
int buflen;
if (stored_bytes != (size_t) -1)
if (dfa->syntax.case_fold)
{
- wchar_t folded[CASE_FOLDED_BUFSIZE];
+ char32_t folded[CASE_FOLDED_BUFSIZE];
int n = case_folded_counterparts (dfa->lex.wctok, folded);
for (int i = 0; i < n; i++)
{
#include <locale.h>
#include <stdlib.h>
#include <string.h>
-#include <wctype.h>
+#if GAWK
+/* Use ISO C 99 API. */
+# include <wctype.h>
+# define char32_t wchar_t
+# define mbrtoc32 mbrtowc
+# define c32tolower towlower
+# define c32toupper towupper
+#else
+/* Use ISO C 11 + gnulib API. */
+# include <uchar.h>
+#endif
/* The sbclen implementation relies on this. */
static_assert (MB_LEN_MAX <= SCHAR_MAX);
static bool
is_using_utf8 (void)
{
- wchar_t wc;
+ char32_t wc;
mbstate_t mbs = {0};
- return mbrtowc (&wc, "\xc4\x80", 2, &mbs) == 2 && wc == 0x100;
+ return mbrtoc32 (&wc, "\xc4\x80", 2, &mbs) == 2 && wc == 0x100;
}
/* Return true if the locale is compatible enough with the C locale so
char c = i;
unsigned char uc = i;
mbstate_t s = {0};
- wchar_t wc;
- size_t len = mbrtowc (&wc, &c, 1, &s);
+ char32_t wc;
+ size_t len = mbrtoc32 (&wc, &c, 1, &s);
localeinfo->sbclen[uc] = len <= 1 ? 1 : - (int) - len;
localeinfo->sbctowc[uc] = len <= 1 ? wc : WEOF;
}
}
-/* The set of wchar_t values C such that there's a useful locale
+/* The set of char32_t values C such that there's a useful locale
somewhere where C != towupper (C) && C != towlower (towupper (C)).
For example, 0x00B5 (U+00B5 MICRO SIGN) is in this table, because
towupper (0x00B5) == 0x039C (U+039C GREEK CAPITAL LETTER MU), and
towlower (0x039C) == 0x03BC (U+03BC GREEK SMALL LETTER MU). */
-static short const lonesome_lower[] =
+static unsigned short int const lonesome_lower[] =
{
0x00B5, 0x0131, 0x017F, 0x01C5, 0x01C8, 0x01CB, 0x01F2, 0x0345,
0x03C2, 0x03D0, 0x03D1, 0x03D5, 0x03D6, 0x03F0, 0x03F1,
stored; this is zero if C is WEOF. */
int
-case_folded_counterparts (wint_t c, wchar_t folded[CASE_FOLDED_BUFSIZE])
+case_folded_counterparts (wint_t c, char32_t folded[CASE_FOLDED_BUFSIZE])
{
int i;
int n = 0;
- wint_t uc = towupper (c);
- wint_t lc = towlower (uc);
+ wint_t uc = c32toupper (c);
+ wint_t lc = c32tolower (uc);
if (uc != c)
folded[n++] = uc;
- if (lc != uc && lc != c && towupper (lc) == uc)
+ if (lc != uc && lc != c && c32toupper (lc) == uc)
folded[n++] = lc;
for (i = 0; i < sizeof lonesome_lower / sizeof *lonesome_lower; i++)
{
wint_t li = lonesome_lower[i];
- if (li != lc && li != uc && li != c && towupper (li) == uc)
+ if (li != lc && li != uc && li != c && c32toupper (li) == uc)
folded[n++] = li;
}
return n;
#include <limits.h>
#include <wchar.h>
+#if GAWK
+/* Use ISO C 99 API. */
+# define char32_t wchar_t
+#else
+/* Use ISO C 11 + gnulib API. */
+# include <uchar.h>
+#endif
struct localeinfo
{
signed char sbclen[UCHAR_MAX + 1];
/* An array indexed by byte values B that contains the corresponding
- wide character (if any) for B if sbclen[B] == 1. WEOF means the
- byte is not a valid single-byte character, i.e., sbclen[B] == -1
+ 32-bit wide character (if any) for B if sbclen[B] == 1. WEOF means
+ the byte is not a valid single-byte character, i.e., sbclen[B] == -1
or -2. */
wint_t sbctowc[UCHAR_MAX + 1];
};
itself. This is a generous upper bound. */
enum { CASE_FOLDED_BUFSIZE = 32 };
-extern int case_folded_counterparts (wint_t, wchar_t[CASE_FOLDED_BUFSIZE]);
+extern int case_folded_counterparts (wint_t, char32_t[CASE_FOLDED_BUFSIZE]);
Depends-on:
assert
+c32isprint
+c32isspace
+c32rtomb
+c32tob
+c32tolower
+c32toupper
c99
ctype
flexmember
idx
locale
+mbrtoc32-regular
regex
stdbool
stddef
stdio
stdlib
string
+uchar
+# The lonesome_lower array requires ISO C 23 semantics for char32_t.
+# But uchar-c23 has a global effect, therefore leave it to each package
+# to enable it.
+#uchar-c23
verify
wchar
-wctype-h
xalloc
xalloc-die
"localeinfo.h"
Link:
+$(LTLIBUNISTRING) when linking with libtool, $(LIBUNISTRING) otherwise
$(MBRTOWC_LIB)
+$(LTLIBC32CONV) when linking with libtool, $(LIBC32CONV) otherwise
License:
GPL
test-dfa-match.sh
check_PROGRAMS += test-dfa-match-aux
-test_dfa_match_aux_LDADD = $(LDADD) $(SETLOCALE_LIB) @LIBINTL@ $(MBRTOWC_LIB)
+test_dfa_match_aux_LDADD = $(LDADD) $(SETLOCALE_LIB) $(LIBUNISTRING) @LIBINTL@ $(MBRTOWC_LIB) $(LIBC32CONV)